diff --git a/.github/workflows/flux.yml b/.github/workflows/flux.yml index da012b1afdb6d..df832a85aaec0 100644 --- a/.github/workflows/flux.yml +++ b/.github/workflows/flux.yml @@ -8,7 +8,7 @@ on: branches: [main] env: - FLUX_VERSION: "6769bb9cbd1216380f542349494efafafd6104a3" + FLUX_VERSION: "1b04d5e6f8247766cfb0f9c6040c2944eb9c87c4" FIXPOINT_VERSION: "nightly" jobs: diff --git a/library/Cargo.lock b/library/Cargo.lock index a07eac016fd53..ccc01aa7de928 100644 --- a/library/Cargo.lock +++ b/library/Cargo.lock @@ -157,9 +157,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.189" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" dependencies = [ "rustc-std-workspace-core", ] diff --git a/library/alloc/src/boxed.rs b/library/alloc/src/boxed.rs index f426345f9fc62..1c5718473a06b 100644 --- a/library/alloc/src/boxed.rs +++ b/library/alloc/src/boxed.rs @@ -53,20 +53,26 @@ //! //! # Memory layout //! -//! For non-zero-sized values, a [`Box`] will use the [`Global`] allocator for its allocation. It is -//! valid to convert both ways between a [`Box`] and a raw pointer allocated with the [`Global`] -//! allocator, given that the [`Layout`] used with the allocator is correct for the type and the raw -//! pointer points to a valid value of the right type. More precisely, a `value: *mut T` that has -//! been allocated with the [`Global`] allocator with `Layout::for_value(&*value)` may be converted -//! into a box using [`Box::::from_raw(value)`]. Conversely, the memory backing a `value: *mut T` -//! obtained from [`Box::::into_raw`] may be deallocated using the [`Global`] allocator with -//! [`Layout::for_value(&*value)`]. +//! For non-zero-sized values, a [`Box`] will use its allocator of type `A` for its allocation. +//! It is valid to convert both ways between a `Box` and a raw pointer allocated with `A`, so long as: +//! - the size of the pointee, as reported by [`size_of_val`], is non-zero, +//! - the [`Layout::for_value`] of the pointee [*fits*](Allocator#memory-fitting) the memory block, and +//! - the pointee is a valid instance of `T` (for the given metadata of the pointer). //! -//! For zero-sized values, the `Box` pointer has to be non-null and sufficiently aligned. The -//! recommended way to build a Box to a ZST if `Box::new` cannot be used is to use -//! [`ptr::NonNull::dangling`]. +//! For zero-sized values, a [`Box`] may use any sufficiently aligned non-null pointer. +//! While it is sound to convert a zero-sized allocation from `A` into a `Box`, as +//! long as it is backed by a valid value of `T`, doing so may lead to memory leaks, as +//! the destructor of [`Box`] will not call [`deallocate`](Allocator::deallocate) +//! on zero-sized values. It is recommended to use [`ptr::dangling`] instead. //! -//! On top of these basic layout requirements, a `Box` must point to a valid value of `T`. +//! Likewise, manually deallocating the raw pointer acquired from a `Box` is only sound +//! if the size of the pointee is non-zero, as zero-sized values may be represented by +//! a pointer that is not [*currently allocated*] by that allocator. It is recommended to +//! pair [`Box::into_raw_with_allocator`] with [`Box::from_raw_in`], rather than +//! doing this. +//! +//! Note that even the [`Global`] allocator might not use dangling pointers for +//! zero-sized allocations, even though *most* implementations do this. //! //! So long as `T: Sized`, a `Box` is guaranteed to be represented //! as a single pointer and is also ABI-compatible with C pointers @@ -172,6 +178,7 @@ //! and it's best to avoid relying on this edition-dependent behavior if you wish to preserve //! compatibility with future versions of the compiler. //! +//! [*currently allocated*]: Allocator#currently-allocated-memory //! [ucg#198]: https://github.com/rust-lang/unsafe-code-guidelines/issues/198 //! [ucg#326]: https://github.com/rust-lang/unsafe-code-guidelines/issues/326 //! [dereferencing]: core::ops::Deref @@ -299,7 +306,7 @@ impl From> for NonNull { // The declaration of the `Box` struct must be kept in sync with the // compiler or ICEs will happen. pub struct Box< - T: ?Sized, + #[rustc_assert_variance(covariant)] T: ?Sized, #[stable(feature = "allocator_api", since = "1.100.0")] A: Allocator = Global, >(BoxRaw, A); @@ -343,12 +350,15 @@ impl Box { /// Allocates memory on the heap and then places `x` into it. /// /// This doesn't actually allocate if `T` is zero-sized. + /// See the [memory layout] section for more details. /// /// # Examples /// /// ``` /// let five = Box::new(5); /// ``` + /// + /// [memory layout]: self#memory-layout #[cfg(not(no_global_oom_handling))] #[inline(always)] #[stable(feature = "rust1", since = "1.0.0")] @@ -435,6 +445,7 @@ impl Box { /// returning an error if the allocation fails /// /// This doesn't actually allocate if `T` is zero-sized. + /// See the [memory layout] section for more details. /// /// # Examples /// @@ -444,6 +455,8 @@ impl Box { /// let five = Box::try_new(5)?; /// # Ok::<(), std::alloc::AllocError>(()) /// ``` + /// + /// [memory layout]: self#memory-layout #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn try_new(x: T) -> Result { @@ -502,6 +515,7 @@ impl Box { /// Allocates memory in the given allocator then places `x` into it. /// /// This doesn't actually allocate if `T` is zero-sized. + /// See the [memory layout] section for more details. /// /// # Examples /// @@ -510,6 +524,8 @@ impl Box { /// /// let five = Box::new_in(5, System); /// ``` + /// + /// [memory layout]: self#memory-layout #[cfg(not(no_global_oom_handling))] #[stable(feature = "allocator_api", since = "1.100.0")] #[must_use] @@ -525,6 +541,7 @@ impl Box { /// returning an error if the allocation fails /// /// This doesn't actually allocate if `T` is zero-sized. + /// See the [memory layout] section for more details. /// /// # Examples /// @@ -536,6 +553,8 @@ impl Box { /// let five = Box::try_new_in(5, System)?; /// # Ok::<(), std::alloc::AllocError>(()) /// ``` + /// + /// [memory layout]: self#memory-layout #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] pub fn try_new_in(x: T, alloc: A) -> Result { @@ -842,6 +861,7 @@ impl Box { /// Allocates memory on the heap then clones `src` into it. /// /// This doesn't actually allocate if `src` is zero-sized. + /// See the [memory layout] section for more details. /// /// # Examples /// @@ -850,6 +870,8 @@ impl Box { /// /// let hello: Box = Box::clone_from_ref("hello"); /// ``` + /// + /// [memory layout]: self#memory-layout #[cfg(not(no_global_oom_handling))] #[unstable(feature = "clone_from_ref", issue = "149075")] #[must_use] @@ -861,6 +883,7 @@ impl Box { /// Allocates memory on the heap then clones `src` into it, returning an error if allocation fails. /// /// This doesn't actually allocate if `src` is zero-sized. + /// See the [memory layout] section for more details. /// /// # Examples /// @@ -870,6 +893,8 @@ impl Box { /// let hello: Box = Box::try_clone_from_ref("hello")?; /// # Ok::<(), std::alloc::AllocError>(()) /// ``` + /// + /// [memory layout]: self#memory-layout #[unstable(feature = "clone_from_ref", issue = "149075")] //#[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] @@ -882,6 +907,7 @@ impl Box { /// Allocates memory in the given allocator then clones `src` into it. /// /// This doesn't actually allocate if `src` is zero-sized. + /// See the [memory layout] section for more details. /// /// # Examples /// @@ -892,6 +918,8 @@ impl Box { /// /// let hello: Box = Box::clone_from_ref_in("hello", System); /// ``` + /// + /// [memory layout]: self#memory-layout #[cfg(not(no_global_oom_handling))] #[unstable(feature = "clone_from_ref", issue = "149075")] //#[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] @@ -908,6 +936,7 @@ impl Box { /// Allocates memory in the given allocator then clones `src` into it, returning an error if allocation fails. /// /// This doesn't actually allocate if `src` is zero-sized. + /// See the [memory layout] section for more details. /// /// # Examples /// @@ -919,6 +948,8 @@ impl Box { /// let hello: Box = Box::try_clone_from_ref_in("hello", System)?; /// # Ok::<(), std::alloc::AllocError>(()) /// ``` + /// + /// [memory layout]: self#memory-layout #[unstable(feature = "clone_from_ref", issue = "149075")] //#[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] #[inline] @@ -1368,9 +1399,16 @@ impl Box { /// memory problems. For example, a double-free may occur if the /// function is called twice on the same raw pointer. /// - /// The raw pointer must point to a block of memory allocated by the global allocator. + /// The exact safety requirements are described in the [memory layout] section, together + /// with recommendations about how to handle the special case of zero-sized allocations + /// correctly. + /// + /// In particular, if the size of the pointee, as reported by [`size_of_val`], is + /// nonzero, then the raw pointer must point to a block of memory allocated by + /// [`Global`]. + /// If the size of the pointee is zero, then the raw pointer can be any + /// sufficiently aligned non-null pointer, e.g. [`ptr::dangling`]. /// - /// The safety conditions are described in the [memory layout] section. /// Note that the [considerations for unsafe code] apply to all `Box` values. /// /// # Examples @@ -1420,9 +1458,16 @@ impl Box { /// memory problems. For example, a double-free may occur if the /// function is called twice on the same `NonNull` pointer. /// - /// The non-null pointer must point to a block of memory allocated by the global allocator. + /// The exact safety requirements are described in the [memory layout] section, together + /// with recommendations about how to handle the special case of zero-sized allocations + /// correctly. + /// + /// In particular, if the size of the pointee, as reported by [`size_of_val`], is + /// nonzero, then the raw pointer must point to a block of memory allocated by + /// [`Global`]. + /// If the size of the pointee is zero, then the raw pointer can be any + /// sufficiently aligned pointer, e.g. [`NonNull::dangling`]. /// - /// The safety conditions are described in the [memory layout] section. /// Note that the [considerations for unsafe code] apply to all `Box` values. /// /// # Examples @@ -1471,6 +1516,11 @@ impl Box { /// [`Box::from_raw`] function, allowing the `Box` destructor to perform /// the cleanup. /// + /// While manually deallocating the pointer through [`deallocate`] + /// is possible, you must only do this if the pointee's size, as reported by + /// [`size_of_val`], is non-zero (the `Box` may be a dangling pointer + /// otherwise). See the [memory layout] section for more details. + /// /// Note: this is an associated function, which means that you have /// to call it as `Box::into_raw(b)` instead of `b.into_raw()`. This /// is so that there is no conflict with a method on the inner type. @@ -1506,6 +1556,7 @@ impl Box { /// ``` /// /// [memory layout]: self#memory-layout + /// [`deallocate`]: Allocator::deallocate #[must_use = "losing the pointer will leak memory"] #[stable(feature = "box_raw", since = "1.4.0")] #[inline] @@ -1533,6 +1584,11 @@ impl Box { /// [`Box::from_non_null`] function, allowing the `Box` destructor to /// perform the cleanup. /// + /// While manually deallocating the pointer through [`deallocate`] + /// is possible, you must only do this if the pointee's size, as reported by + /// [`size_of_val`], is non-zero (the `Box` may be a dangling pointer + /// otherwise). See the [memory layout] section for more details. + /// /// Note: this is an associated function, which means that you have /// to call it as `Box::into_non_null(b)` instead of `b.into_non_null()`. /// This is so that there is no conflict with a method on the inner type. @@ -1567,6 +1623,7 @@ impl Box { /// ``` /// /// [memory layout]: self#memory-layout + /// [`deallocate`]: Allocator::deallocate #[must_use = "losing the pointer will leak memory"] #[stable(feature = "box_vec_non_null", since = "1.99.0")] #[inline] @@ -1585,9 +1642,8 @@ impl Box { /// /// After calling this function, the raw pointer is owned by the /// resulting `Box`. Specifically, the `Box` destructor will call - /// the destructor of `T` and free the allocated memory. For this - /// to be safe, the memory must have been allocated in accordance - /// with the [memory layout] used by `Box` . + /// the destructor of `T` and free the allocated memory, except + /// in the cases described in the [memory layout] section. /// /// # Safety /// @@ -1595,9 +1651,16 @@ impl Box { /// memory problems. For example, a double-free may occur if the /// function is called twice on the same raw pointer. /// - /// The raw pointer must point to a block of memory allocated by `alloc`. + /// The exact safety requirements are described in the [memory layout] section, together + /// with recommendations about how to handle the special case of zero-sized allocations + /// correctly. + /// + /// In particular, if the size of the pointee, as reported by [`size_of_val`], is + /// nonzero, then the raw pointer must point to a block of memory allocated by + /// `alloc`. + /// If the size of the pointee is zero, then the raw pointer can be any + /// sufficiently aligned non-null pointer, e.g. [`ptr::dangling`]. /// - /// The safety conditions are described in the [memory layout] section. /// Note that the [considerations for unsafe code] apply to all `Box` values. /// /// # Examples @@ -1641,9 +1704,8 @@ impl Box { /// /// After calling this function, the `NonNull` pointer is owned by /// the resulting `Box`. Specifically, the `Box` destructor will call - /// the destructor of `T` and free the allocated memory. For this - /// to be safe, the memory must have been allocated in accordance - /// with the [memory layout] used by `Box` . + /// the destructor of `T` and free the allocated memory, except + /// in the cases described in the [memory layout] section. /// /// # Safety /// @@ -1651,9 +1713,16 @@ impl Box { /// memory problems. For example, a double-free may occur if the /// function is called twice on the same raw pointer. /// - /// The non-null pointer must point to a block of memory allocated by `alloc`. + /// The exact safety requirements are described in the [memory layout] section, together + /// with recommendations about how to handle the special case of zero-sized allocations + /// correctly. + /// + /// In particular, if the size of the pointee, as reported by [`size_of_val`], is + /// nonzero, then the raw pointer must point to a block of memory allocated by + /// `alloc`. + /// If the size of the pointee is zero, then the raw pointer can be any + /// sufficiently aligned pointer, e.g. [`NonNull::dangling`]. /// - /// The safety conditions are described in the [memory layout] section. /// Note that the [considerations for unsafe code] apply to all `Box` values. /// /// # Examples @@ -1702,6 +1771,11 @@ impl Box { /// [`Box::from_raw_in`] function, allowing the `Box` destructor to perform /// the cleanup. /// + /// While manually deallocating the pointer through [`deallocate`] + /// is possible, you must only do this if the pointee's size, as reported by + /// [`size_of_val`], is non-zero (the `Box` may be a dangling pointer + /// otherwise). See the [memory layout] section for more details. + /// /// Note: this is an associated function, which means that you have /// to call it as `Box::into_raw_with_allocator(b)` instead of `b.into_raw_with_allocator()`. This /// is so that there is no conflict with a method on the inner type. @@ -1732,6 +1806,7 @@ impl Box { /// ``` /// /// [memory layout]: self#memory-layout + /// [`deallocate`]: Allocator::deallocate #[must_use = "losing the pointer will leak memory"] #[stable(feature = "allocator_api", since = "1.100.0")] #[rustc_const_unstable(feature = "const_heap", issue = "79597")] @@ -1761,6 +1836,11 @@ impl Box { /// [`Box::from_non_null_in`] function, allowing the `Box` destructor to /// perform the cleanup. /// + /// While manually deallocating the pointer through [`deallocate`] + /// is possible, you must only do this if the pointee's size, as reported by + /// [`size_of_val`], is non-zero (the `Box` may be a dangling pointer + /// otherwise). See the [memory layout] section for more details. + /// /// Note: this is an associated function, which means that you have /// to call it as `Box::into_non_null_with_allocator(b)` instead of /// `b.into_non_null_with_allocator()`. This is so that there is no @@ -1790,6 +1870,7 @@ impl Box { /// ``` /// /// [memory layout]: self#memory-layout + /// [`deallocate`]: Allocator::deallocate #[must_use = "losing the pointer will leak memory"] #[stable(feature = "allocator_api", since = "1.100.0")] #[inline] diff --git a/library/alloc/src/bstr.rs b/library/alloc/src/byte_str.rs similarity index 81% rename from library/alloc/src/bstr.rs rename to library/alloc/src/byte_str.rs index 59909ed1db9b2..55caf4012b21c 100644 --- a/library/alloc/src/bstr.rs +++ b/library/alloc/src/byte_str.rs @@ -4,9 +4,9 @@ #![cfg(not(no_global_oom_handling))] use core::borrow::{Borrow, BorrowMut}; -#[unstable(feature = "bstr", issue = "134915")] -pub use core::bstr::ByteStr; -use core::bstr::{impl_partial_eq, impl_partial_eq_n, impl_partial_eq_ord}; +#[unstable(feature = "byte_str", issue = "134915")] +pub use core::byte_str::ByteStr; +use core::byte_str::{impl_partial_eq, impl_partial_eq_n, impl_partial_eq_ord}; use core::cmp::Ordering; use core::ops::{ Deref, DerefMut, DerefPure, Index, IndexMut, Range, RangeFrom, RangeFull, RangeInclusive, @@ -33,7 +33,7 @@ use crate::vec::Vec; /// need to round-trip whatever data the user provides. /// /// A `ByteString` owns its contents and can grow and shrink, like a `Vec` or `String`. For a -/// borrowed byte string, see [`ByteStr`](../../std/bstr/struct.ByteStr.html). +/// borrowed byte string, see [`ByteStr`](../../std/byte_str/struct.ByteStr.html). /// /// `ByteString` implements `Deref` to `&Vec`, so all methods available on `&Vec` are /// available on `ByteString`. Similarly, `ByteString` implements `DerefMut` to `&mut Vec`, @@ -41,7 +41,7 @@ use crate::vec::Vec; /// /// The `Debug` and `Display` implementations for `ByteString` are the same as those for `ByteStr`, /// showing invalid UTF-8 as hex escapes or the Unicode replacement character, respectively. -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[repr(transparent)] #[derive(Clone, Default)] #[doc(alias = "BString")] @@ -70,7 +70,7 @@ impl ByteString { /// implementation for types that implement `Display`, and the trait version /// will use the Unicode replacement character rather than returning a /// `Result` and allowing for the possibility of the content not being UTF-8. - #[unstable(feature = "bstr_to_string", issue = "134915")] + #[unstable(feature = "byte_str_to_string", issue = "134915")] #[rustc_allow_incoherent_impl] pub fn to_string(&self) -> Result { // Avoid allocating a copy of the contents for invalid UTF-8 @@ -82,7 +82,7 @@ impl ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl Deref for ByteString { type Target = Vec; @@ -92,7 +92,7 @@ impl Deref for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl DerefMut for ByteString { #[inline] fn deref_mut(&mut self) -> &mut Self::Target { @@ -103,7 +103,7 @@ impl DerefMut for ByteString { #[unstable(feature = "deref_pure_trait", issue = "87121")] unsafe impl DerefPure for ByteString {} -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl fmt::Debug for ByteString { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -111,7 +111,7 @@ impl fmt::Debug for ByteString { } } -#[unstable(feature = "bstr_to_string", issue = "134915")] +#[unstable(feature = "byte_str_to_string", issue = "134915")] impl fmt::Display for ByteString { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -119,7 +119,7 @@ impl fmt::Display for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl AsRef<[u8]> for ByteString { #[inline] fn as_ref(&self) -> &[u8] { @@ -127,7 +127,7 @@ impl AsRef<[u8]> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl AsRef for ByteString { #[inline] fn as_ref(&self) -> &ByteStr { @@ -135,7 +135,7 @@ impl AsRef for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl AsMut<[u8]> for ByteString { #[inline] fn as_mut(&mut self) -> &mut [u8] { @@ -143,7 +143,7 @@ impl AsMut<[u8]> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl AsMut for ByteString { #[inline] fn as_mut(&mut self) -> &mut ByteStr { @@ -151,7 +151,7 @@ impl AsMut for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl Borrow<[u8]> for ByteString { #[inline] fn borrow(&self) -> &[u8] { @@ -159,7 +159,7 @@ impl Borrow<[u8]> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl Borrow for ByteString { #[inline] fn borrow(&self) -> &ByteStr { @@ -170,7 +170,7 @@ impl Borrow for ByteString { // `impl Borrow for Vec` omitted to avoid inference failures // `impl Borrow for String` omitted to avoid inference failures -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl BorrowMut<[u8]> for ByteString { #[inline] fn borrow_mut(&mut self) -> &mut [u8] { @@ -178,7 +178,7 @@ impl BorrowMut<[u8]> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl BorrowMut for ByteString { #[inline] fn borrow_mut(&mut self) -> &mut ByteStr { @@ -190,7 +190,7 @@ impl BorrowMut for ByteString { // Omitted due to inference failures // -// #[unstable(feature = "bstr", issue = "134915")] +// #[unstable(feature = "byte_str", issue = "134915")] // impl<'a, const N: usize> From<&'a [u8; N]> for ByteString { // #[inline] // fn from(s: &'a [u8; N]) -> Self { @@ -198,7 +198,7 @@ impl BorrowMut for ByteString { // } // } // -// #[unstable(feature = "bstr", issue = "134915")] +// #[unstable(feature = "byte_str", issue = "134915")] // impl From<[u8; N]> for ByteString { // #[inline] // fn from(s: [u8; N]) -> Self { @@ -206,7 +206,7 @@ impl BorrowMut for ByteString { // } // } // -// #[unstable(feature = "bstr", issue = "134915")] +// #[unstable(feature = "byte_str", issue = "134915")] // impl<'a> From<&'a [u8]> for ByteString { // #[inline] // fn from(s: &'a [u8]) -> Self { @@ -214,7 +214,7 @@ impl BorrowMut for ByteString { // } // } // -// #[unstable(feature = "bstr", issue = "134915")] +// #[unstable(feature = "byte_str", issue = "134915")] // impl From> for ByteString { // #[inline] // fn from(s: Vec) -> Self { @@ -222,7 +222,7 @@ impl BorrowMut for ByteString { // } // } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl From for Vec { #[inline] fn from(s: ByteString) -> Self { @@ -232,7 +232,7 @@ impl From for Vec { // Omitted due to inference failures // -// #[unstable(feature = "bstr", issue = "134915")] +// #[unstable(feature = "byte_str", issue = "134915")] // impl<'a> From<&'a str> for ByteString { // #[inline] // fn from(s: &'a str) -> Self { @@ -240,7 +240,7 @@ impl From for Vec { // } // } // -// #[unstable(feature = "bstr", issue = "134915")] +// #[unstable(feature = "byte_str", issue = "134915")] // impl From for ByteString { // #[inline] // fn from(s: String) -> Self { @@ -248,7 +248,7 @@ impl From for Vec { // } // } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl<'a> From<&'a ByteStr> for ByteString { #[inline] fn from(s: &'a ByteStr) -> Self { @@ -256,7 +256,7 @@ impl<'a> From<&'a ByteStr> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl<'a> From for Cow<'a, ByteStr> { #[inline] fn from(s: ByteString) -> Self { @@ -264,7 +264,7 @@ impl<'a> From for Cow<'a, ByteStr> { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl<'a> From<&'a ByteString> for Cow<'a, ByteStr> { #[inline] fn from(s: &'a ByteString) -> Self { @@ -272,7 +272,7 @@ impl<'a> From<&'a ByteString> for Cow<'a, ByteStr> { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl FromIterator for ByteString { #[inline] fn from_iter>(iter: I) -> Self { @@ -280,7 +280,7 @@ impl FromIterator for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl FromIterator for ByteString { #[inline] fn from_iter>(iter: I) -> Self { @@ -288,7 +288,7 @@ impl FromIterator for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl<'a> FromIterator<&'a str> for ByteString { #[inline] fn from_iter>(iter: I) -> Self { @@ -296,7 +296,7 @@ impl<'a> FromIterator<&'a str> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl<'a> FromIterator<&'a [u8]> for ByteString { #[inline] fn from_iter>(iter: I) -> Self { @@ -308,7 +308,7 @@ impl<'a> FromIterator<&'a [u8]> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl<'a> FromIterator<&'a ByteStr> for ByteString { #[inline] fn from_iter>(iter: I) -> Self { @@ -320,7 +320,7 @@ impl<'a> FromIterator<&'a ByteStr> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl FromIterator for ByteString { #[inline] fn from_iter>(iter: I) -> Self { @@ -332,7 +332,7 @@ impl FromIterator for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl FromStr for ByteString { type Err = !; @@ -342,7 +342,7 @@ impl FromStr for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl Index for ByteString { type Output = u8; @@ -352,7 +352,7 @@ impl Index for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl Index for ByteString { type Output = ByteStr; @@ -362,7 +362,7 @@ impl Index for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl Index> for ByteString { type Output = ByteStr; @@ -372,7 +372,7 @@ impl Index> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl Index> for ByteString { type Output = ByteStr; @@ -382,7 +382,7 @@ impl Index> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl Index> for ByteString { type Output = ByteStr; @@ -392,7 +392,7 @@ impl Index> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl Index> for ByteString { type Output = ByteStr; @@ -402,7 +402,7 @@ impl Index> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl Index> for ByteString { type Output = ByteStr; @@ -412,7 +412,7 @@ impl Index> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl IndexMut for ByteString { #[inline] fn index_mut(&mut self, idx: usize) -> &mut u8 { @@ -420,7 +420,7 @@ impl IndexMut for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl IndexMut for ByteString { #[inline] fn index_mut(&mut self, _: RangeFull) -> &mut ByteStr { @@ -428,7 +428,7 @@ impl IndexMut for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl IndexMut> for ByteString { #[inline] fn index_mut(&mut self, r: Range) -> &mut ByteStr { @@ -436,7 +436,7 @@ impl IndexMut> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl IndexMut> for ByteString { #[inline] fn index_mut(&mut self, r: RangeInclusive) -> &mut ByteStr { @@ -444,7 +444,7 @@ impl IndexMut> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl IndexMut> for ByteString { #[inline] fn index_mut(&mut self, r: RangeFrom) -> &mut ByteStr { @@ -452,7 +452,7 @@ impl IndexMut> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl IndexMut> for ByteString { #[inline] fn index_mut(&mut self, r: RangeTo) -> &mut ByteStr { @@ -460,7 +460,7 @@ impl IndexMut> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl IndexMut> for ByteString { #[inline] fn index_mut(&mut self, r: RangeToInclusive) -> &mut ByteStr { @@ -468,7 +468,7 @@ impl IndexMut> for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl hash::Hash for ByteString { #[inline] fn hash(&self, state: &mut H) { @@ -476,10 +476,10 @@ impl hash::Hash for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl Eq for ByteString {} -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl PartialEq for ByteString { #[inline] fn eq(&self, other: &ByteString) -> bool { @@ -489,7 +489,7 @@ impl PartialEq for ByteString { macro_rules! impl_partial_eq_ord_cow { ($lhs:ty, $rhs:ty) => { - #[unstable(feature = "bstr", issue = "134915")] + #[unstable(feature = "byte_str", issue = "134915")] impl PartialEq<$rhs> for $lhs { #[inline] fn eq(&self, other: &$rhs) -> bool { @@ -498,7 +498,7 @@ macro_rules! impl_partial_eq_ord_cow { } } - #[unstable(feature = "bstr", issue = "134915")] + #[unstable(feature = "byte_str", issue = "134915")] impl PartialEq<$lhs> for $rhs { #[inline] fn eq(&self, other: &$lhs) -> bool { @@ -507,7 +507,7 @@ macro_rules! impl_partial_eq_ord_cow { } } - #[unstable(feature = "bstr", issue = "134915")] + #[unstable(feature = "byte_str", issue = "134915")] impl PartialOrd<$rhs> for $lhs { #[inline] fn partial_cmp(&self, other: &$rhs) -> Option { @@ -516,7 +516,7 @@ macro_rules! impl_partial_eq_ord_cow { } } - #[unstable(feature = "bstr", issue = "134915")] + #[unstable(feature = "byte_str", issue = "134915")] impl PartialOrd<$lhs> for $rhs { #[inline] fn partial_cmp(&self, other: &$lhs) -> Option { @@ -549,7 +549,7 @@ impl_partial_eq_ord_cow!(ByteString, Cow<'_, ByteStr>); impl_partial_eq_ord_cow!(ByteString, Cow<'_, str>); impl_partial_eq_ord_cow!(ByteString, Cow<'_, [u8]>); -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl Ord for ByteString { #[inline] fn cmp(&self, other: &ByteString) -> Ordering { @@ -557,7 +557,7 @@ impl Ord for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl PartialOrd for ByteString { #[inline] fn partial_cmp(&self, other: &ByteString) -> Option { @@ -565,7 +565,7 @@ impl PartialOrd for ByteString { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl ToOwned for ByteStr { type Owned = ByteString; @@ -575,7 +575,7 @@ impl ToOwned for ByteStr { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl TryFrom for String { type Error = crate::string::FromUtf8Error; @@ -585,7 +585,7 @@ impl TryFrom for String { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl<'a> TryFrom<&'a ByteString> for &'a str { type Error = crate::str::Utf8Error; @@ -597,7 +597,7 @@ impl<'a> TryFrom<&'a ByteString> for &'a str { // Additional impls for `ByteStr` that require types from `alloc`: -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl Clone for Box { #[inline] fn clone(&self) -> Self { @@ -605,7 +605,7 @@ impl Clone for Box { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl<'a> From<&'a ByteStr> for Cow<'a, ByteStr> { #[inline] fn from(s: &'a ByteStr) -> Self { @@ -613,7 +613,7 @@ impl<'a> From<&'a ByteStr> for Cow<'a, ByteStr> { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl From> for Box { #[inline] fn from(s: Box<[u8]>) -> Box { @@ -622,7 +622,7 @@ impl From> for Box { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl From> for Box<[u8]> { #[inline] fn from(s: Box) -> Box<[u8]> { @@ -631,7 +631,7 @@ impl From> for Box<[u8]> { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[cfg(not(no_rc))] impl From> for Rc { #[inline] @@ -641,7 +641,7 @@ impl From> for Rc { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[cfg(not(no_rc))] impl From> for Rc<[u8]> { #[inline] @@ -651,7 +651,7 @@ impl From> for Rc<[u8]> { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[cfg(all(not(no_rc), not(no_sync), target_has_atomic = "ptr"))] impl From> for Arc { #[inline] @@ -661,7 +661,7 @@ impl From> for Arc { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[cfg(all(not(no_rc), not(no_sync), target_has_atomic = "ptr"))] impl From> for Arc<[u8]> { #[inline] @@ -679,7 +679,7 @@ impl_partial_eq_ord_cow!(&ByteStr, Cow<'_, ByteStr>); impl_partial_eq_ord_cow!(&ByteStr, Cow<'_, str>); impl_partial_eq_ord_cow!(&ByteStr, Cow<'_, [u8]>); -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl<'a> TryFrom<&'a ByteStr> for String { type Error = core::str::Utf8Error; @@ -698,7 +698,7 @@ impl ByteStr { /// implementation for types that implement `Display`, and the trait version /// will use the Unicode replacement character rather than returning a /// `Result` and allowing for the possibility of the content not being UTF-8. - #[unstable(feature = "bstr_to_string", issue = "134915")] + #[unstable(feature = "byte_str_to_string", issue = "134915")] #[rustc_allow_incoherent_impl] pub fn to_string(&self) -> Result { // Avoid allocating a copy of the contents for invalid UTF-8 diff --git a/library/alloc/src/collections/btree/map.rs b/library/alloc/src/collections/btree/map.rs index 109beed7de947..89e20e03afe83 100644 --- a/library/alloc/src/collections/btree/map.rs +++ b/library/alloc/src/collections/btree/map.rs @@ -205,16 +205,13 @@ pub struct BTreeMap< #[stable(feature = "btree_drop", since = "1.7.0")] unsafe impl<#[may_dangle] K, #[may_dangle] V, A: AllocatorClone> Drop for BTreeMap { fn drop(&mut self) { - // Skip `into_iter` for an empty map: `dying_next` is too costly to inline, so the - // empty drop isn't optimised away (see #161375). - if self.root.is_some() { - // SAFETY: `self` is not used after this and none of its fields are dropped again: - // `alloc` is `ManuallyDrop` and `root` has no drop glue. - drop(unsafe { ptr::read(self) }.into_iter()) + if let Some(root) = self.root.take() { + // SAFETY: The tree won't be used after this point. + // The allocator has not been moved out and will be moved into `drop_tree`. + unsafe { root.into_dying().drop_tree(ManuallyDrop::take(&mut self.alloc)) }; } else { - // SAFETY: With no root there are no nodes to free, so only the allocator needs - // dropping. `self` is not used after this, and `alloc` is dropped only here. - unsafe { ManuallyDrop::drop(&mut self.alloc) } + // SAFETY: The allocator has not been moved out and won't be used after this point. + unsafe { ManuallyDrop::drop(&mut self.alloc) }; } } } diff --git a/library/alloc/src/collections/btree/map/tests.rs b/library/alloc/src/collections/btree/map/tests.rs index a920254872e19..c4c9148e0abfb 100644 --- a/library/alloc/src/collections/btree/map/tests.rs +++ b/library/alloc/src/collections/btree/map/tests.rs @@ -2542,6 +2542,114 @@ fn test_into_iter_drop_leak_height_1() { } } +#[test] +fn test_drop_map_elements_multi_level() { + for size in [1, MIN_INSERTS_HEIGHT_1, MIN_INSERTS_HEIGHT_2] { + let keys = Vec::from_iter((0..size).map(CrashTestDummy::new)); + let vals = Vec::from_iter((0..size).map(CrashTestDummy::new)); + let map = { + let mut map = BTreeMap::new(); + + (0..size).map(|i| (keys[i].spawn(Panic::Never), vals[i].spawn(Panic::Never))).for_each( + |(k, v)| { + map.insert(k, v); + }, + ); + map + }; + + drop(map); + + for i in 0..size { + assert_eq!(keys[i].dropped(), 1); + assert_eq!(vals[i].dropped(), 1); + } + } +} + +#[test] +fn test_drop_empty_map_allocator() { + use core::alloc::{AllocError, Allocator, AllocatorClone, Layout}; + use core::cell::Cell; + use core::ptr::NonNull; + + struct CountedAllocator<'a>(&'a Cell); + + impl Clone for CountedAllocator<'_> { + fn clone(&self) -> Self { + Self(self.0) + } + } + + impl Drop for CountedAllocator<'_> { + fn drop(&mut self) { + self.0.set(self.0.get() + 1); + } + } + + unsafe impl Allocator for CountedAllocator<'_> { + fn allocate(&self, layout: Layout) -> Result, AllocError> { + Global.allocate(layout) + } + + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + unsafe { Global.deallocate(ptr, layout) } + } + } + + unsafe impl AllocatorClone for CountedAllocator<'_> {} + + let dropped = Cell::new(0); + let map = BTreeMap::::new_in(CountedAllocator(&dropped)); + assert!(map.root.is_none()); + + drop(map); + + assert_eq!(dropped.get(), 1); +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore = "test requires unwinding support")] +fn test_drop_map_elements_multi_level_panic() { + for size in [1, MIN_INSERTS_HEIGHT_1, MIN_INSERTS_HEIGHT_2] { + for panic_point in [0, size / 2, size - 1] { + for panic_in_key in [true, false] { + let keys = Vec::from_iter((0..size).map(CrashTestDummy::new)); + let vals = Vec::from_iter((0..size).map(CrashTestDummy::new)); + let map = { + let mut map = BTreeMap::new(); + + (0..size) + .map(|i| { + let key_panic = if panic_in_key && i == panic_point { + Panic::InDrop + } else { + Panic::Never + }; + let val_panic = if !panic_in_key && i == panic_point { + Panic::InDrop + } else { + Panic::Never + }; + (keys[i].spawn(key_panic), vals[i].spawn(val_panic)) + }) + .for_each(|(k, v)| { + map.insert(k, v); + }); + map + }; + + catch_unwind(move || drop(map)).unwrap_err(); + + for i in 0..size { + assert_eq!(keys[i].dropped(), 1); + assert_eq!(vals[i].dropped(), 1); + } + } + } + } +} + #[test] fn test_into_keys() { let map = BTreeMap::from([(1, 'a'), (2, 'b'), (3, 'c')]); diff --git a/library/alloc/src/collections/btree/navigate.rs b/library/alloc/src/collections/btree/navigate.rs index 8b91aacda0b2e..d0a87cf542858 100644 --- a/library/alloc/src/collections/btree/navigate.rs +++ b/library/alloc/src/collections/btree/navigate.rs @@ -852,3 +852,70 @@ impl NodeRef NodeRef { + /// # Safety + /// - `self` must be the root. + /// - The tree shouldn't be accessed anymore. + pub(super) unsafe fn drop_tree(self, alloc: A) { + struct DropGuard { + node: Option>, + alloc: A, + } + + impl Drop for DropGuard { + fn drop(&mut self) { + if let Some(cur) = self.node.take() { + let mut next = + // SAFETY: When unwinding reached this guard, all kvs in cur + // have already been dropped in `drop_kvs`, while the node itself has not + // yet been deallocated. + unsafe { deallocate_and_next_postorder_node(cur, self.alloc.clone()) }; + + while let Some(mut node) = next { + // SAFETY: Every node only calls the function once + unsafe { node.drop_kvs() }; + next = + // SAFETY: `drop_kvs()` above only dropped kvs and the node is still accessible, + // and also each node is deallocated exactly once. + unsafe { deallocate_and_next_postorder_node(node, self.alloc.clone()) } + } + } + } + } + /// Deallocates `node` and returns the next node in postorder traversal + /// + /// # Safety + /// + /// The node must be initialized, all kvs in node must have already + /// been dropped or moved out and its children must have already been deallocated. + /// Also the node can't be touched anymore. + unsafe fn deallocate_and_next_postorder_node( + node: NodeRef, + alloc: A, + ) -> Option> { + // SAFETY: guaranteed by the caller + let parent = unsafe { node.deallocate_and_ascend(alloc) }?; + + Some(match parent.right_kv() { + Ok(kv) => kv.right_edge().descend().first_leaf_edge().into_node().forget_type(), + Err(last_edge) => last_edge.into_node().forget_type(), + }) + } + + let node = self.first_leaf_edge().into_node().forget_type(); + let mut guard = DropGuard { node: Some(node), alloc }; + loop { + // SAFETY: all kvs of each node are dropped once + unsafe { guard.node.as_mut().unwrap().drop_kvs() }; + // SAFETY: Nodes are deallocated in postorder, and each node is deallocated exactly once. + if let Some(next) = unsafe { + deallocate_and_next_postorder_node(guard.node.take().unwrap(), guard.alloc.clone()) + } { + guard.node = Some(next); + } else { + return; + } + } + } +} diff --git a/library/alloc/src/collections/btree/node.rs b/library/alloc/src/collections/btree/node.rs index 47c3fb1d79c9d..2144efc5d7d4d 100644 --- a/library/alloc/src/collections/btree/node.rs +++ b/library/alloc/src/collections/btree/node.rs @@ -429,6 +429,45 @@ impl NodeRef { } ret } + /// Drops all kvs in this node without deallocating the node itself + /// + /// # Safety + /// + /// - kvs in the node must be initialized and must not be accessed after this call. + /// - No kvs in the node can be accessed through any other path while this function is running. + pub(super) unsafe fn drop_kvs(&mut self) { + struct DropGuard<'a, K, V> { + node: &'a NodeRef, + idx: usize, + } + impl<'a, K, V> Drop for DropGuard<'a, K, V> { + fn drop(&mut self) { + let len = self.node.len(); + for i in self.idx..len { + // SAFETY: i < len and the kv has been initialized and + // it won't use the i-th kv more than once. + unsafe { + let kv = Handle::new_kv(ptr::read(self.node), i); + kv.drop_key_val(); + } + } + } + } + let len = self.len(); + for idx in 0..len { + // If `kv.drop_key_val()` panics, the guard would drop + // the rest and skip the panicked kv. + let guard = DropGuard { node: self, idx: idx + 1 }; + // SAFETY: idx < len and the kv has been initialized and + // it won't touch the idx-th more than once + unsafe { + let kv = Handle::new_kv(ptr::read(self), idx); + kv.drop_key_val(); + } + + core::mem::forget(guard); + } + } } impl<'a, K, V, Type> NodeRef, K, V, Type> { diff --git a/library/alloc/src/collections/linked_list/tests.rs b/library/alloc/src/collections/linked_list/tests.rs index a51af6345e205..1fe63070f1a15 100644 --- a/library/alloc/src/collections/linked_list/tests.rs +++ b/library/alloc/src/collections/linked_list/tests.rs @@ -304,6 +304,7 @@ fn test_clone_from() { #[test] #[cfg_attr(target_os = "emscripten", ignore)] +#[cfg_attr(target_os = "wasi", ignore)] fn test_send() { let n = list_from(&[1, 2, 3]); thread::spawn(move || { diff --git a/library/alloc/src/collections/vec_deque/tests.rs b/library/alloc/src/collections/vec_deque/tests.rs index d9cf56de7bb6b..422f72885145c 100644 --- a/library/alloc/src/collections/vec_deque/tests.rs +++ b/library/alloc/src/collections/vec_deque/tests.rs @@ -1431,3 +1431,55 @@ fn extract_if_pred_panic_leak() { assert_eq!(DROPS.get(), 2); // 0 and 1 assert_eq!(q.len(), 6); } + +#[test] +fn test_custom_allocator_balanced() { + use core::alloc::{AllocError, Allocator, Layout}; + use core::cell::Cell; + use core::ptr::NonNull; + + struct CountingAlloc<'a> { + live: &'a Cell, + dropped: &'a Cell, + } + + impl<'a> Drop for CountingAlloc<'a> { + fn drop(&mut self) { + self.dropped.set(self.dropped.get() + 1); + } + } + + unsafe impl Allocator for CountingAlloc<'_> { + fn allocate(&self, layout: Layout) -> Result, AllocError> { + let ptr = Global.allocate(layout)?; + self.live.set(self.live.get() + 1); + Ok(ptr) + } + + unsafe fn deallocate(&self, ptr: NonNull, layout: Layout) { + self.live.set(self.live.get() - 1); + unsafe { Global.deallocate(ptr, layout) } + } + } + + let live = Cell::new(0); + let dropped = Cell::new(0); + { + let mut d = VecDeque::new_in(CountingAlloc { live: &live, dropped: &dropped }); + assert_eq!(live.get(), 0); + + for i in 0..4 { + d.push_back(i); + } + d.pop_front(); + d.pop_front(); + d.push_back(4); + d.push_back(5); + d.push_back(6); + assert_eq!(d.iter().copied().collect::>(), [2, 3, 4, 5, 6]); + assert_eq!(live.get(), 1); + assert_eq!(dropped.get(), 0); + } + assert_eq!(live.get(), 0); + assert_eq!(dropped.get(), 1); +} diff --git a/library/alloc/src/io/copy.rs b/library/alloc/src/io/copy.rs index 798bd2c1ec0a0..f414078d3a0b5 100644 --- a/library/alloc/src/io/copy.rs +++ b/library/alloc/src/io/copy.rs @@ -75,6 +75,7 @@ pub enum CopyState { /// descriptors if possible. /// /// Note that platform-specific behavior may change in the future. +#[doc(alias = "copy_file_range", alias = "sendfile", alias = "splice")] #[stable(feature = "rust1", since = "1.0.0")] pub fn copy(reader: &mut R, writer: &mut W) -> Result where diff --git a/library/alloc/src/lib.rs b/library/alloc/src/lib.rs index 261830c518972..f499a7f2326de 100644 --- a/library/alloc/src/lib.rs +++ b/library/alloc/src/lib.rs @@ -97,8 +97,8 @@ #![feature(async_fn_traits)] #![feature(async_iterator)] #![feature(borrowed_buf_init)] -#![feature(bstr)] -#![feature(bstr_internals)] +#![feature(byte_str)] +#![feature(byte_str_internals)] #![feature(can_vector)] #![feature(case_ignorable)] #![feature(cast_maybe_uninit)] @@ -159,7 +159,6 @@ #![feature(pin_coerce_unsized_trait)] #![feature(ptr_alignment_type)] #![feature(ptr_cast_slice)] -#![feature(ptr_internals)] #![feature(ptr_metadata)] #![feature(random)] #![feature(raw_os_error_ty)] @@ -259,8 +258,8 @@ pub mod alloc; // to allow code to have `use boxed::Box;` declarations. pub mod borrow; pub mod boxed; -#[unstable(feature = "bstr", issue = "134915")] -pub mod bstr; +#[unstable(feature = "byte_str", issue = "134915")] +pub mod byte_str; pub mod collections; #[cfg(all(not(no_rc), not(no_sync), not(no_global_oom_handling)))] pub mod ffi; diff --git a/library/alloc/src/rcs/arc.rs b/library/alloc/src/rcs/arc.rs index 8ae3c094b2910..3ad02c09b428a 100644 --- a/library/alloc/src/rcs/arc.rs +++ b/library/alloc/src/rcs/arc.rs @@ -213,7 +213,7 @@ macro_rules! acquire { /// /// Sharing some immutable data between threads: /// -/// ``` +/// ```ignore-wasm /// use std::sync::Arc; /// use std::thread; /// @@ -232,7 +232,7 @@ macro_rules! acquire { /// /// [`AtomicUsize`]: core::sync::atomic::AtomicUsize "sync::atomic::AtomicUsize" /// -/// ``` +/// ```ignore-wasm /// use std::sync::Arc; /// use std::sync::atomic::{AtomicUsize, Ordering}; /// use std::thread; @@ -263,7 +263,7 @@ macro_rules! acquire { note = "consider using `Arc::clone`" )] pub struct Arc< - T: ?Sized, + #[rustc_assert_variance(covariant)] T: ?Sized, // FIXME: When stabilizing this parameter, the `AllocatorNightly` // bound on the `From` impl *must* be weaked to `Allocator`, // as `Box` is fundamental! @@ -1091,7 +1091,7 @@ impl Arc { /// # Examples /// /// Minimal example demonstrating the guarantee that `Arc::into_inner` gives. - /// ``` + /// ```ignore-wasm /// use std::sync::Arc; /// /// let x = Arc::new(3); @@ -1114,7 +1114,7 @@ impl Arc { /// ``` /// /// A more practical example demonstrating the need for `Arc::into_inner`: - /// ``` + /// ```ignore-wasm /// use std::sync::Arc; /// /// // Definition of a simple singly linked list using `Arc`: diff --git a/library/alloc/src/rcs/rc.rs b/library/alloc/src/rcs/rc.rs index 696bd733b992d..26538f2123ffc 100644 --- a/library/alloc/src/rcs/rc.rs +++ b/library/alloc/src/rcs/rc.rs @@ -283,7 +283,7 @@ use crate::vec::Vec; // repr(align(2)) (forcing alignment to at least 2) is required because usize // has 1-byte alignment on AVR. #[repr(C, align(2))] -struct RcInner { +struct RcInner<#[rustc_assert_variance(covariant)] T: ?Sized> { strong: Cell, weak: Cell, value: T, @@ -323,7 +323,7 @@ fn rc_inner_layout_for_value_layout(layout: Layout) -> Layout { )] pub struct Rc< - T: ?Sized, + #[rustc_assert_variance(covariant)] T: ?Sized, // FIXME: When stabilizing this parameter, the `AllocatorNightly` // bound on the `From` impl *must* be weaked to `Allocator`, // as `Box` is fundamental! diff --git a/library/alloc/src/string.rs b/library/alloc/src/string.rs index 03c07b15712bb..bb8301e53d34f 100644 --- a/library/alloc/src/string.rs +++ b/library/alloc/src/string.rs @@ -940,6 +940,10 @@ impl String { /// into a `String` with the [`from_raw_parts`] function, allowing /// the destructor to perform the cleanup. /// + /// If `capacity == 0`, then `pointer` may be dangling. Otherwise, + /// it will always be valid for reads and writes over `capacity` + /// bytes, and point to the start of the string. + /// /// [`from_raw_parts`]: String::from_raw_parts /// /// # Examples @@ -3414,6 +3418,10 @@ impl TryFrom> for String { } } +/// Note that all methods of this trait append to the `String`'s existing +/// contents; they never truncate or replace what is already there. In +/// particular, [`fmt::Write::write_str`] uses [`push_str`][String::push_str] +/// and [`fmt::Write::write_char`] uses [`push`][String::push]. #[cfg(not(no_global_oom_handling))] #[stable(feature = "rust1", since = "1.0.0")] impl fmt::Write for String { diff --git a/library/alloc/src/vec/mod.rs b/library/alloc/src/vec/mod.rs index 3cc9eba5f1c4d..847cebd476322 100644 --- a/library/alloc/src/vec/mod.rs +++ b/library/alloc/src/vec/mod.rs @@ -51,25 +51,50 @@ //! //! # Memory layout //! -//! When the type is non-zero-sized and the capacity is nonzero, [`Vec`] uses the [`Global`] -//! allocator for its allocation. It is valid to convert both ways between such a [`Vec`] and a raw -//! pointer allocated with the [`Global`] allocator, provided that the [`Layout`] used with the -//! allocator is correct for a sequence of `capacity` elements of the type, and the first `len` -//! values pointed to by the raw pointer are valid. More precisely, a `ptr: *mut T` that has been -//! allocated with the [`Global`] allocator with [`Layout::array::(capacity)`][Layout::array] may -//! be converted into a vec using -//! [`Vec::::from_raw_parts(ptr, len, capacity)`](Vec::from_raw_parts). Conversely, the memory -//! backing a `value: *mut T` obtained from [`Vec::::as_mut_ptr`] may be deallocated using the -//! [`Global`] allocator with the same layout. +//! A [`Vec`] conceptually consists of a raw pointer `ptr` to the start of the allocation, a +//! length `len`, a capacity `cap`, and an allocator `alloc`. +//! While the exact fields and their order are not specified, a [`Vec`] can be converted to and +//! from these parts using [`Vec::into_raw_parts_with_allocator`], [`Vec::from_raw_parts_in`], +//! [`Vec::into_raw_parts`], and [`Vec::from_raw_parts`], as well as their [`NonNull`]-based +//! versions. //! -//! For zero-sized types (ZSTs), or when the capacity is zero, the `Vec` pointer must be non-null -//! and sufficiently aligned. The recommended way to build a `Vec` of ZSTs if [`vec!`] cannot be -//! used is to use [`ptr::NonNull::dangling`]. +//! The parts of a [`Vec`] have the following invariants: //! +//! * If [`size_of::()`](size_of) and `cap` are non-zero, then: +//! - `ptr` must point to the start of a memory block [*currently allocated*] via `alloc`. +//! - [`Layout::array::(cap)`] must [*fit*] the memory block of `ptr`. In particular, +//! * `T` must have the same alignment as the [`Layout`] `ptr` was allocated with, and +//! * `size_of::() * cap` must be at least the size of the [`Layout`] `ptr` was +//! allocated with, and at most the size of the allocation returned the allocator. +//! - `size_of::() * cap` must be at most [`isize::MAX`]. +//! See the safety documentation of [`pointer::offset`]. +//! * If [`size_of::()`](size_of) or `cap` is zero, then `ptr` must be any sufficiently aligned non-null pointer. +//! +//! NOTE: `ptr` will not be passed to the allocator for deallocation by the destructor of [`Vec`] +//! in this case. Thus, it is unwise to use a pointer acquired from the allocator for this, as +//! overallocated memory may end up being leaked. Use [`ptr::dangling`] instead. +//! * The following must always be satisfied: +//! - `len` needs to be less than or equal to `cap`. +//! - At `ptr`, there are at least `len` valid values of type `T`, laid out contiguously +//! (like an array). These values are considered owned by the [`Vec`]. +//! +//! It is sound to create a [`Vec`] from parts that satisfy these invariants. +//! +//! Conversely, the memory backing `ptr` obtained from [`Vec::as_mut_ptr`] or +//! [`Vec::into_raw_parts_with_allocator`]-like methods may be deallocated +//! using the allocator, with layout [`Layout::array::(cap)`], provided +//! that [`size_of::()`](size_of) or `cap` is non-zero. +//! To automatically handle the case where the allocation is zero-sized, +//! it is recommended to use [`Vec::from_raw_parts_in`], instead of manually +//! deallocating. +//! +//! [*fit*]: crate::alloc::Allocator#memory-fitting +//! [*currently allocated*]: crate::alloc::Allocator#currently-allocated-memory +//! [`allocate`]: crate::alloc::Allocator::allocate //! [`push`]: Vec::push //! [`ptr::NonNull::dangling`]: NonNull::dangling //! [`Layout`]: crate::alloc::Layout -//! [Layout::array]: crate::alloc::Layout::array +//! [`Layout::array::(cap)`]: crate::alloc::Layout::array #![stable(feature = "rust1", since = "1.0.0")] @@ -303,35 +328,13 @@ mod sve_retain; /// /// # Guarantees /// +/// The following section is to be read as an addendum of the +/// [memory layout](self#memory-layout) section. +/// /// Due to its incredibly fundamental nature, `Vec` makes a lot of guarantees /// about its design. This ensures that it's as low-overhead as possible in /// the general case, and can be correctly manipulated in primitive ways -/// by unsafe code. Note that these guarantees refer to an unqualified `Vec`. -/// If additional type parameters are added (e.g., to support custom allocators), -/// overriding their defaults may change the behavior. -/// -/// Most fundamentally, `Vec` is and always will be a (pointer, capacity, length) -/// triplet. No more, no less. The order of these fields is completely -/// unspecified, and you should use the appropriate methods to modify these. -/// The pointer will never be null, so this type is null-pointer-optimized. -/// -/// However, the pointer might not actually point to allocated memory. In particular, -/// if you construct a `Vec` with capacity 0 via [`Vec::new`], [`vec![]`][`vec!`], -/// [`Vec::with_capacity(0)`][`Vec::with_capacity`], or by calling [`shrink_to_fit`] -/// on an empty Vec, it will not allocate memory. Similarly, if you store zero-sized -/// types inside a `Vec`, it will not allocate space for them. *Note that in this case -/// the `Vec` might not report a [`capacity`] of 0*. `Vec` will allocate if and only -/// if [size_of::\]\() * [capacity]\() > 0. In general, `Vec`'s allocation -/// details are very subtle --- if you intend to allocate memory using a `Vec` -/// and use it for something else (either to pass to unsafe code, or to build your -/// own memory-backed collection), be sure to deallocate this memory by using -/// `from_raw_parts` to recover the `Vec` and then dropping it. -/// -/// If a `Vec` *has* allocated memory, then the memory it points to is on the heap -/// (as defined by the allocator Rust is configured to use by default), and its -/// pointer points to [`len`] initialized, contiguous elements in order (what -/// you would see if you coerced it to a slice), followed by [capacity] - [len] -/// logically uninitialized, contiguous elements. +/// by unsafe code. /// /// A vector containing the elements `'a'` and `'b'` with capacity 4 can be /// visualized as below. The top part is the `Vec` struct, it contains a @@ -469,7 +472,7 @@ impl Vec { /// /// The vector will be able to hold at least `capacity` elements without /// reallocating. This method is allowed to allocate for more elements than - /// `capacity`. If `capacity` is zero, the vector will not allocate. + /// `capacity`. If `capacity` is zero, the vector [will not allocate]. /// /// It is important to note that although the returned vector has the /// minimum *capacity* specified, the vector will have a zero *length*. For @@ -479,11 +482,12 @@ impl Vec { /// If it is important to know the exact allocated capacity of a `Vec`, /// always use the [`capacity`] method after construction. /// - /// For `Vec` where `T` is a zero-sized type, there will be no allocation - /// and the capacity will always be `usize::MAX`. + /// For `Vec` where `T` is a zero-sized type, the vector + /// [will not allocate], and the capacity will always be `usize::MAX`. /// /// [Capacity and reallocation]: #capacity-and-reallocation /// [`capacity`]: Vec::capacity + /// [will not allocate]: self#memory-layout /// /// # Panics /// @@ -529,12 +533,14 @@ impl Vec { /// /// The vector will be able to hold at least `capacity` elements without /// reallocating. This method is allowed to allocate for more elements than - /// `capacity`. If `capacity` is zero, the vector will not allocate. + /// `capacity`. If `capacity` is zero, the vector [will not allocate]. /// /// # Errors /// /// Returns an error if the capacity exceeds `isize::MAX` _bytes_, /// or if the allocator reports allocation failure. + /// + /// [will not allocate]: self#memory-layout #[inline] #[unstable(feature = "try_with_capacity", issue = "91913")] pub fn try_with_capacity(capacity: usize) -> Result { @@ -545,34 +551,10 @@ impl Vec { /// /// # Safety /// - /// This is highly unsafe, due to the number of invariants that aren't - /// checked: - /// - /// * If `T` is not a zero-sized type and the capacity is nonzero, `ptr` must have - /// been allocated using the global allocator, such as via the [`alloc::alloc`] - /// function. If `T` is a zero-sized type or the capacity is zero, `ptr` need - /// only be non-null and aligned. - /// * `T` needs to have the same alignment as what `ptr` was allocated with, - /// if the pointer is required to be allocated. - /// (`T` having a less strict alignment is not sufficient, the alignment really - /// needs to be equal to satisfy the [`dealloc`] requirement that memory must be - /// allocated and deallocated with the same layout.) - /// * The size of `T` times the `capacity` (i.e. the allocated size in bytes), if - /// nonzero, needs to be the same size as the pointer was allocated with. - /// (Because similar to alignment, [`dealloc`] must be called with the same - /// layout `size`.) - /// * `length` needs to be less than or equal to `capacity`. - /// * The first `length` values must be properly initialized values of type `T`. - /// * `capacity` needs to be the capacity that the pointer was allocated with, - /// if the pointer is required to be allocated. - /// * The allocated size in bytes must be no larger than `isize::MAX`. - /// See the safety documentation of [`pointer::offset`]. - /// - /// These requirements are always upheld by any `ptr` that has been allocated - /// via `Vec`. Other allocation sources are allowed if the invariants are - /// upheld. - /// - /// Violating these may cause problems like corrupting the allocator's + /// The parts must satisfy the invariants described in the [memory layout] section, + /// with allocator [`Global`]. + /// + /// Violating them may cause problems like corrupting the allocator's /// internal data structures. For example it is normally **not** safe /// to build a `Vec` from a pointer to a C `char` array with length /// `size_t`, doing so is only safe if the array was initially allocated by @@ -591,8 +573,7 @@ impl Vec { /// function. /// /// [`String`]: crate::string::String - /// [`alloc::alloc`]: crate::alloc::alloc - /// [`dealloc`]: crate::alloc::GlobalAlloc::dealloc + /// [memory layout]: self#memory-layout /// /// # Examples /// @@ -652,29 +633,10 @@ impl Vec { /// /// # Safety /// - /// This is highly unsafe, due to the number of invariants that aren't - /// checked: - /// - /// * `ptr` must have been allocated using the global allocator, such as via - /// the [`alloc::alloc`] function. - /// * `T` needs to have the same alignment as what `ptr` was allocated with. - /// (`T` having a less strict alignment is not sufficient, the alignment really - /// needs to be equal to satisfy the [`dealloc`] requirement that memory must be - /// allocated and deallocated with the same layout.) - /// * The size of `T` times the `capacity` (i.e. the allocated size in bytes) needs - /// to be the same size as the pointer was allocated with. (Because similar to - /// alignment, [`dealloc`] must be called with the same layout `size`.) - /// * `length` needs to be less than or equal to `capacity`. - /// * The first `length` values must be properly initialized values of type `T`. - /// * `capacity` needs to be the capacity that the pointer was allocated with. - /// * The allocated size in bytes must be no larger than `isize::MAX`. - /// See the safety documentation of [`pointer::offset`]. - /// - /// These requirements are always upheld by any `ptr` that has been allocated - /// via `Vec`. Other allocation sources are allowed if the invariants are - /// upheld. - /// - /// Violating these may cause problems like corrupting the allocator's + /// The parts must satisfy the invariants described in the [memory layout] section, + /// with allocator [`Global`]. + /// + /// Violating them may cause problems like corrupting the allocator's /// internal data structures. For example it is normally **not** safe /// to build a `Vec` from a pointer to a C `char` array with length /// `size_t`, doing so is only safe if the array was initially allocated by @@ -693,8 +655,7 @@ impl Vec { /// function. /// /// [`String`]: crate::string::String - /// [`alloc::alloc`]: crate::alloc::alloc - /// [`dealloc`]: crate::alloc::GlobalAlloc::dealloc + /// [memory layout]: self#memory-layout /// /// # Examples /// @@ -917,7 +878,7 @@ const impl Vec { /// /// The vector will be able to hold at least `capacity` elements without /// reallocating. This method is allowed to allocate for more elements than - /// `capacity`. If `capacity` is zero, the vector will not allocate. + /// `capacity`. If `capacity` is zero, the vector [will not allocate]. /// /// It is important to note that although the returned vector has the /// minimum *capacity* specified, the vector will have a zero *length*. For @@ -927,11 +888,12 @@ const impl Vec { /// If it is important to know the exact allocated capacity of a `Vec`, /// always use the [`capacity`] method after construction. /// - /// For `Vec` where `T` is a zero-sized type, there will be no allocation - /// and the capacity will always be `usize::MAX`. + /// For `Vec` where `T` is a zero-sized type, the vector + /// [will not allocate], and the capacity will always be `usize::MAX`. /// /// [Capacity and reallocation]: #capacity-and-reallocation /// [`capacity`]: Vec::capacity + /// [will not allocate]: self#memory-layout /// /// # Panics /// @@ -1069,12 +1031,14 @@ impl Vec { /// /// The vector will be able to hold at least `capacity` elements without /// reallocating. This method is allowed to allocate for more elements than - /// `capacity`. If `capacity` is zero, the vector will not allocate. + /// `capacity`. If `capacity` is zero, the vector [will not allocate]. /// /// # Errors /// /// Returns an error if the capacity exceeds `isize::MAX` _bytes_, /// or if the allocator reports allocation failure. + /// + /// [will not allocate]: self#memory-layout #[inline] #[unstable(feature = "allocator_ext", issue = "163177", implied_by = "allocator_api")] // #[unstable(feature = "try_with_capacity", issue = "91913")] @@ -1087,28 +1051,9 @@ impl Vec { /// /// # Safety /// - /// This is highly unsafe, due to the number of invariants that aren't - /// checked: - /// - /// * `ptr` must be [*currently allocated*] via the given allocator `alloc`. - /// * `T` needs to have the same alignment as what `ptr` was allocated with. - /// (`T` having a less strict alignment is not sufficient, the alignment really - /// needs to be equal to satisfy the [`dealloc`] requirement that memory must be - /// allocated and deallocated with the same layout.) - /// * The size of `T` times the `capacity` (i.e. the allocated size in bytes) needs - /// to be the same size as the pointer was allocated with. (Because similar to - /// alignment, [`dealloc`] must be called with the same layout `size`.) - /// * `length` needs to be less than or equal to `capacity`. - /// * The first `length` values must be properly initialized values of type `T`. - /// * `capacity` needs to [*fit*] the layout size that the pointer was allocated with. - /// * The allocated size in bytes must be no larger than `isize::MAX`. - /// See the safety documentation of [`pointer::offset`]. - /// - /// These requirements are always upheld by any `ptr` that has been allocated - /// via `Vec`. Other allocation sources are allowed if the invariants are - /// upheld. - /// - /// Violating these may cause problems like corrupting the allocator's + /// The parts must satisfy the invariants described in the [memory layout] section. + /// + /// Violating them may cause problems like corrupting the allocator's /// internal data structures. For example it is **not** safe /// to build a `Vec` from a pointer to a C `char` array with length `size_t`. /// It's also not safe to build one from a `Vec` and its length, because @@ -1123,9 +1068,8 @@ impl Vec { /// function. /// /// [`String`]: crate::string::String - /// [`dealloc`]: crate::alloc::GlobalAlloc::dealloc - /// [*currently allocated*]: crate::alloc::Allocator#currently-allocated-memory - /// [*fit*]: crate::alloc::Allocator#memory-fitting + /// [`allocate`]: crate::alloc::Allocator::allocate + /// [memory layout]: self#memory-layout /// /// # Examples /// @@ -1201,28 +1145,9 @@ impl Vec { /// /// # Safety /// - /// This is highly unsafe, due to the number of invariants that aren't - /// checked: - /// - /// * `ptr` must be [*currently allocated*] via the given allocator `alloc`. - /// * `T` needs to have the same alignment as what `ptr` was allocated with. - /// (`T` having a less strict alignment is not sufficient, the alignment really - /// needs to be equal to satisfy the [`dealloc`] requirement that memory must be - /// allocated and deallocated with the same layout.) - /// * The size of `T` times the `capacity` (i.e. the allocated size in bytes) needs - /// to be the same size as the pointer was allocated with. (Because similar to - /// alignment, [`dealloc`] must be called with the same layout `size`.) - /// * `length` needs to be less than or equal to `capacity`. - /// * The first `length` values must be properly initialized values of type `T`. - /// * `capacity` needs to [*fit*] the layout size that the pointer was allocated with. - /// * The allocated size in bytes must be no larger than `isize::MAX`. - /// See the safety documentation of [`pointer::offset`]. - /// - /// These requirements are always upheld by any `ptr` that has been allocated - /// via `Vec`. Other allocation sources are allowed if the invariants are - /// upheld. - /// - /// Violating these may cause problems like corrupting the allocator's + /// The parts must satisfy the invariants described in the [memory layout] section. + /// + /// Violating them may cause problems like corrupting the allocator's /// internal data structures. For example it is **not** safe /// to build a `Vec` from a pointer to a C `char` array with length `size_t`. /// It's also not safe to build one from a `Vec` and its length, because @@ -1238,8 +1163,7 @@ impl Vec { /// /// [`String`]: crate::string::String /// [`dealloc`]: crate::alloc::GlobalAlloc::dealloc - /// [*currently allocated*]: crate::alloc::Allocator#currently-allocated-memory - /// [*fit*]: crate::alloc::Allocator#memory-fitting + /// [memory layout]: self#memory-layout /// /// # Examples /// @@ -1899,7 +1823,7 @@ impl Vec { } /// Returns a raw pointer to the vector's buffer, or a dangling raw pointer - /// valid for zero sized reads if the vector didn't allocate. + /// valid for zero sized reads if the vector [didn't allocate]. /// /// The caller must ensure that the vector outlives the pointer this /// function returns, or else it will end up dangling. @@ -1951,6 +1875,7 @@ impl Vec { /// [`as_mut_ptr`]: Vec::as_mut_ptr /// [`as_ptr`]: Vec::as_ptr /// [`as_non_null`]: Vec::as_non_null + /// [didn't allocate]: self#memory-layout #[stable(feature = "vec_as_ptr", since = "1.37.0")] #[rustc_const_stable(feature = "const_vec_string_slice", since = "1.87.0")] #[rustc_never_returns_null_ptr] @@ -1963,7 +1888,7 @@ impl Vec { } /// Returns a raw mutable pointer to the vector's buffer, or a dangling - /// raw pointer valid for zero sized reads if the vector didn't allocate. + /// raw pointer valid for zero sized reads if the vector [didn't allocate]. /// /// The caller must ensure that the vector outlives the pointer this /// function returns, or else it will end up dangling. @@ -2035,6 +1960,7 @@ impl Vec { /// [`as_non_null`]: Vec::as_non_null /// [`dealloc`]: crate::alloc::GlobalAlloc::dealloc /// [`ManuallyDrop`]: core::mem::ManuallyDrop + /// [didn't allocate]: self#memory-layout #[stable(feature = "vec_as_ptr", since = "1.37.0")] #[rustc_const_stable(feature = "const_vec_string_slice", since = "1.87.0")] #[rustc_never_returns_null_ptr] @@ -2047,7 +1973,7 @@ impl Vec { } /// Returns a `NonNull` pointer to the vector's buffer, or a dangling - /// `NonNull` pointer valid for zero sized reads if the vector didn't allocate. + /// `NonNull` pointer valid for zero sized reads if the vector [didn't allocate]. /// /// The caller must ensure that the vector outlives the pointer this /// function returns, or else it will end up dangling. @@ -2102,6 +2028,7 @@ impl Vec { /// [`as_mut_ptr`]: Vec::as_mut_ptr /// [`as_ptr`]: Vec::as_ptr /// [`as_non_null`]: Vec::as_non_null + /// [didn't allocate]: self#memory-layout #[unstable(feature = "vec_as_non_null", issue = "157843")] #[rustc_const_unstable(feature = "vec_as_non_null", issue = "157843")] #[rustc_as_ptr] @@ -2713,7 +2640,24 @@ impl Vec { if first_duplicate_idx == len { return; } + // SAFETY: first_duplicate_idx always in range [1..len) + unsafe { + self.dedup_tail(same_bucket, first_duplicate_idx); + } + } + /// Continues deduplication after the first duplicate is found. + /// + /// The function treats `self[first_duplicate]` as the first duplicated + /// element and drops it. + /// + /// # Safety + /// + /// `1 <= first_duplicate < self.len()` + unsafe fn dedup_tail(&mut self, mut same_bucket: F, first_duplicate: usize) + where + F: FnMut(&mut T, &mut T) -> bool, + { /* INVARIANT: vec.len() > read > write > write-1 >= 0 */ struct FillGapOnDrop<'a, T, A: core::alloc::Allocator> { /* Offset of the element we want to check if it is duplicate */ @@ -2760,16 +2704,20 @@ impl Vec { } } + let start = self.as_mut_ptr(); + let len = self.len(); + /* Drop items while going through Vec, it should be more efficient than * doing slice partition_dedup + truncate */ // Construct gap first and then drop item to avoid memory corruption if `T::drop` panics. let mut gap = - FillGapOnDrop { read: first_duplicate_idx + 1, write: first_duplicate_idx, vec: self }; - // SAFETY: we checked that first_duplicate_idx in bounds before. + FillGapOnDrop { read: first_duplicate + 1, write: first_duplicate, vec: self }; + + // SAFETY: By this function's safety contract, `first_duplicate < self.len()`. // If drop panics, `gap` would remove this item without drop. unsafe { - ptr::drop_in_place(start.add(first_duplicate_idx)); + ptr::drop_in_place(start.add(first_duplicate)); } // SAFETY: Because of the invariant, read_ptr, prev_ptr and write_ptr @@ -3788,8 +3736,142 @@ impl Vec { #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn dedup(&mut self) { - self.dedup_by(|a, b| a == b) + // Vectorize the process of finding first duplicate if possible + if let Some(chunk_size) = dedup_simd_chunk_size::() { + // Unlike dedup_by, we don't guarantee the order + // and time in which we call `PartialEq::eq`. + let Some(first_duplicate) = find_first_duplicate(self.as_slice(), chunk_size) else { + return; + }; + + // SAFETY: `1 <= first_duplicate < self.len()` + unsafe { + self.dedup_tail(|a, b| a == b, first_duplicate); + } + } else { + self.dedup_by(|a, b| a == b); + } + } +} + +const fn dedup_simd_chunk_size() -> Option { + // The minimal SIMD width on supported platforms + #[allow(unused)] + const SIMD_WIDTH: usize = 16; + let size = size_of::(); + + // We only have SIMD implementations for these element sizes. + if !matches!(size, 1 | 2 | 4 | 8) { + return None; + } + + let simd_width: Option = cfg_select! { + all(target_arch = "arm", target_feature = "neon") => { + // Packed 64-bit integer lane equality is unavailable on arm neon + if size == 8 { None } else { Some(SIMD_WIDTH) } + } + + all(target_arch = "aarch64", target_feature = "neon") => { + Some(SIMD_WIDTH) + } + + all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse4.1" + ) => Some(SIMD_WIDTH), + + all( + any(target_arch = "x86", target_arch = "x86_64"), + target_feature = "sse2" + ) => { + // Packed 64-bit integer lane equality is only available on sse4.1 + if size == 8 { None } else { Some(SIMD_WIDTH) } + } + + all( + any(target_arch = "riscv32", target_arch = "riscv64"), + target_feature = "v" + ) => Some(SIMD_WIDTH), + + all( + any(target_arch = "loongarch32", target_arch = "loongarch64"), + target_feature = "lsx" + ) => Some(SIMD_WIDTH), + + _ => None, + }; + + if let Some(simd_width) = simd_width { Some(simd_width * 2 / size) } else { None } +} + +fn find_first_duplicate(v: &[T], chunk_size: usize) -> Option { + let len = v.len(); + if len <= 1 { + return None; } + + let mut i = 1; + + // The scalar fallback path if the length is not long enough + if len < chunk_size + 1 { + while i < len { + // It keeps the same comparison order as `dedup_tail` + // + // SAFETY: i always in range [1..len) + if unsafe { v.get_unchecked(i) == v.get_unchecked(i - 1) } { + return Some(i); + } + i += 1; + } + return None; + } + + /// # Safety + /// - `start > 0` and `start + chunk_size <= v.len()` + /// - `0 < chunk_size <= 32` + #[inline(always)] + unsafe fn find_duplicate( + v: &[T], + start: usize, + chunk_size: usize, + ) -> Option { + let mut mask: u32 = 0; + debug_assert!( + u32::BITS as usize >= chunk_size, + "the bit width of mask must be greater than or equal to chunk_size" + ); + for j in 0..chunk_size { + // SAFETY: start + j always in range [1..v.len()) + let eq = unsafe { v.get_unchecked(start + j) == v.get_unchecked(start + j - 1) }; + mask |= (eq as u32) << j; + } + if mask != 0 { + return Some(start + mask.trailing_zeros() as usize); + } + None + } + + // This loop performs chunk_size comparisons in parallel without early exit, + // which is what allows for SIMD optimization. + while i + chunk_size <= len { + // SAFETY: `i >= 1` and `i + chunk_size <= len` + if let Some(duplicate) = unsafe { find_duplicate(v, i, chunk_size) } { + return Some(duplicate); + } + i += chunk_size; + } + + // This may check the same elements multiple times. + // However we don't guarantee `PartialEq::eq` is only called once per pair. + if i < len { + let start = len - chunk_size; + // SAFETY: `len - chunk_size >= 1` and `start + chunk_size <= len` + if let Some(duplicate) = unsafe { find_duplicate(v, start, chunk_size) } { + return Some(duplicate); + } + } + + None } //////////////////////////////////////////////////////////////////////////////// diff --git a/library/alloctests/benches/vec.rs b/library/alloctests/benches/vec.rs index 656164da72084..a4e151ed42bf7 100644 --- a/library/alloctests/benches/vec.rs +++ b/library/alloctests/benches/vec.rs @@ -688,137 +688,176 @@ fn bench_dedup_slice_truncate(b: &mut Bencher, sz: usize) { }); } -// Measures performance of Vec::dedup on random data. -fn bench_vec_dedup_random(b: &mut Bencher, sz: usize) { - let mut template = vec![0u32; sz]; - b.bytes = size_of_val(template.as_slice()) as u64; - random_sorted_fill(0x43, &mut template); - - let mut vec = template.clone(); - b.iter(|| { - let vec = black_box(&mut vec); - vec.dedup(); - black_box(vec.first()); - let vec = black_box(vec); - vec.clear(); - vec.extend_from_slice(&template); - }); -} - -// Measures performance of Vec::dedup when there is no items removed -fn bench_vec_dedup_none(b: &mut Bencher, sz: usize) { - let mut template = vec![0u32; sz]; - b.bytes = size_of_val(template.as_slice()) as u64; - template.chunks_exact_mut(2).for_each(|w| { - w[0] = black_box(0); - w[1] = black_box(5); - }); - - let mut vec = template.clone(); - b.iter(|| { - let vec = black_box(&mut vec); - vec.dedup(); - black_box(vec.first()); - // Unlike other benches of `dedup` - // this doesn't reinitialize vec - // because we measure how efficient dedup is - // when no memory written - }); -} - -// Measures performance of Vec::dedup when there is all items removed -fn bench_vec_dedup_all(b: &mut Bencher, sz: usize) { - let mut template = vec![0u32; sz]; - b.bytes = size_of_val(template.as_slice()) as u64; - template.iter_mut().for_each(|w| { - *w = black_box(0); - }); - - let mut vec = template.clone(); - b.iter(|| { - let vec = black_box(&mut vec); - vec.dedup(); - black_box(vec.first()); - let vec = black_box(vec); - vec.clear(); - vec.extend_from_slice(&template); - }); -} - #[bench] fn bench_dedup_slice_truncate_100(b: &mut Bencher) { bench_dedup_slice_truncate(b, 100); } -#[bench] -fn bench_dedup_random_100(b: &mut Bencher) { - bench_vec_dedup_random(b, 100); -} - -#[bench] -fn bench_dedup_none_100(b: &mut Bencher) { - bench_vec_dedup_none(b, 100); -} - -#[bench] -fn bench_dedup_all_100(b: &mut Bencher) { - bench_vec_dedup_all(b, 100); -} - #[bench] fn bench_dedup_slice_truncate_1000(b: &mut Bencher) { bench_dedup_slice_truncate(b, 1000); } #[bench] -fn bench_dedup_random_1000(b: &mut Bencher) { - bench_vec_dedup_random(b, 1000); +fn bench_dedup_slice_truncate_10000(b: &mut Bencher) { + bench_dedup_slice_truncate(b, 10000); } - #[bench] -fn bench_dedup_none_1000(b: &mut Bencher) { - bench_vec_dedup_none(b, 1000); +fn bench_dedup_slice_truncate_100000(b: &mut Bencher) { + bench_dedup_slice_truncate(b, 100000); } -#[bench] -fn bench_dedup_all_1000(b: &mut Bencher) { - bench_vec_dedup_all(b, 1000); +#[derive(Clone, Copy)] +enum DedupDist { + Random, + None, + All, } -#[bench] -fn bench_dedup_slice_truncate_10000(b: &mut Bencher) { - bench_dedup_slice_truncate(b, 10000); +trait DedupBenchValue { + fn bench_bytes(&self) -> usize; } -#[bench] -fn bench_dedup_random_10000(b: &mut Bencher) { - bench_vec_dedup_random(b, 10000); + +macro_rules! dedup_bench_value_by_size { + ($($ty:ty),+ $(,)?) => { + $( + impl DedupBenchValue for $ty { + fn bench_bytes(&self) -> usize { + size_of_val(self) + } + } + )+ + }; } -#[bench] -fn bench_dedup_none_10000(b: &mut Bencher) { - bench_vec_dedup_none(b, 10000); +dedup_bench_value_by_size!(u8, u16, u32, u64, [u64; 4]); + +impl DedupBenchValue for String { + fn bench_bytes(&self) -> usize { + self.len() + } } -#[bench] -fn bench_dedup_all_10000(b: &mut Bencher) { - bench_vec_dedup_all(b, 10000); +fn bench_vec_dedup( + b: &mut Bencher, + dist: DedupDist, + sz: usize, + make: impl Fn(u32) -> T, +) { + let template: Vec = match dist { + DedupDist::Random => { + let mut seeds = vec![0u32; sz]; + random_sorted_fill(0x43, &mut seeds); + let mut v: Vec = seeds.into_iter().map(make).collect(); + v.sort(); + v + } + DedupDist::None => { + (0..sz).map(|i| make(black_box(if i % 2 == 0 { 0 } else { 5 }))).collect() + } + DedupDist::All => (0..sz).map(|_| make(black_box(0))).collect(), + }; + b.bytes = template.iter().map(DedupBenchValue::bench_bytes).sum::() as u64; + let mut vec = template.clone(); + b.iter(|| { + let vec = black_box(&mut vec); + vec.dedup(); + black_box(vec.first()); + // Unlike other benches of `dedup` + // `DedupDist::None` doesn't reinitialize vec + // because we measure how efficient dedup is + // when no memory written + if !matches!(dist, DedupDist::None) { + black_box(vec).clone_from(&template); + } + }); } -#[bench] -fn bench_dedup_slice_truncate_100000(b: &mut Bencher) { - bench_dedup_slice_truncate(b, 100000); +fn dedup_gen_string(x: u32) -> String { + const DIGITS: usize = 10; + let mut s = String::with_capacity(LEN); + s.extend(std::iter::repeat_n('p', LEN - DIGITS)); + s.push_str(&format!("{x:0DIGITS$}")); + s } -#[bench] -fn bench_dedup_random_100000(b: &mut Bencher) { - bench_vec_dedup_random(b, 100000); + +fn dedup_gen_wide(x: u32) -> [u64; 4] { + [x as u64, (x as u64) << 8, (x as u64) << 16, (x as u64) << 24] } -#[bench] -fn bench_dedup_none_100000(b: &mut Bencher) { - bench_vec_dedup_none(b, 100000); +macro_rules! dedup_typed { + ($($name:ident: $gen:expr, $dist:ident, $sz:literal;)*) => { + $( + #[bench] + fn $name(b: &mut Bencher) { + bench_vec_dedup(b, DedupDist::$dist, $sz, $gen); + } + )* + }; } -#[bench] -fn bench_dedup_all_100000(b: &mut Bencher) { - bench_vec_dedup_all(b, 100000); +dedup_typed! { + bench_dedup_random_100: |x| x, Random, 100; + bench_dedup_none_100: |x| x, None, 100; + bench_dedup_all_100: |x| x, All, 100; + bench_dedup_random_1000: |x| x, Random, 1000; + bench_dedup_none_1000: |x| x, None, 1000; + bench_dedup_all_1000: |x| x, All, 1000; + bench_dedup_random_10000: |x| x, Random, 10000; + bench_dedup_none_10000: |x| x, None, 10000; + bench_dedup_all_10000: |x| x, All, 10000; + bench_dedup_random_100000: |x| x, Random, 100000; + bench_dedup_none_100000: |x| x, None, 100000; + bench_dedup_all_100000: |x| x, All, 100000; + bench_dedup_string_short_random_100: dedup_gen_string::<10>, Random, 100; + bench_dedup_string_short_none_100: dedup_gen_string::<10>, None, 100; + bench_dedup_string_short_all_100: dedup_gen_string::<10>, All, 100; + bench_dedup_string_short_random_1000: dedup_gen_string::<10>, Random, 1000; + bench_dedup_string_short_none_1000: dedup_gen_string::<10>, None, 1000; + bench_dedup_string_short_all_1000: dedup_gen_string::<10>, All, 1000; + bench_dedup_string_short_random_10000: dedup_gen_string::<10>, Random, 10000; + bench_dedup_string_short_none_10000: dedup_gen_string::<10>, None, 10000; + bench_dedup_string_short_all_10000: dedup_gen_string::<10>, All, 10000; + bench_dedup_string_medium_random_100: dedup_gen_string::<64>, Random, 100; + bench_dedup_string_medium_none_100: dedup_gen_string::<64>, None, 100; + bench_dedup_string_medium_all_100: dedup_gen_string::<64>, All, 100; + bench_dedup_string_medium_random_1000: dedup_gen_string::<64>, Random, 1000; + bench_dedup_string_medium_none_1000: dedup_gen_string::<64>, None, 1000; + bench_dedup_string_medium_all_1000: dedup_gen_string::<64>, All, 1000; + bench_dedup_string_medium_random_10000: dedup_gen_string::<64>, Random, 10000; + bench_dedup_string_medium_none_10000: dedup_gen_string::<64>, None, 10000; + bench_dedup_string_medium_all_10000: dedup_gen_string::<64>, All, 10000; + bench_dedup_string_long_random_100: dedup_gen_string::<1024>, Random, 100; + bench_dedup_string_long_none_100: dedup_gen_string::<1024>, None, 100; + bench_dedup_string_long_all_100: dedup_gen_string::<1024>, All, 100; + bench_dedup_string_long_random_1000: dedup_gen_string::<1024>, Random, 1000; + bench_dedup_string_long_none_1000: dedup_gen_string::<1024>, None, 1000; + bench_dedup_string_long_all_1000: dedup_gen_string::<1024>, All, 1000; + bench_dedup_string_long_random_10000: dedup_gen_string::<1024>, Random, 10000; + bench_dedup_string_long_none_10000: dedup_gen_string::<1024>, None, 10000; + bench_dedup_string_long_all_10000: dedup_gen_string::<1024>, All, 10000; + bench_dedup_u8_random_1000: |x| x as u8, Random, 1000; + bench_dedup_u8_none_1000: |x| x as u8, None, 1000; + bench_dedup_u8_all_1000: |x| x as u8, All, 1000; + bench_dedup_u8_random_10000: |x| x as u8, Random, 10000; + bench_dedup_u8_none_10000: |x| x as u8, None, 10000; + bench_dedup_u8_all_10000: |x| x as u8, All, 10000; + bench_dedup_u16_random_1000: |x| x as u16, Random, 1000; + bench_dedup_u16_none_1000: |x| x as u16, None, 1000; + bench_dedup_u16_all_1000: |x| x as u16, All, 1000; + bench_dedup_u16_random_10000: |x| x as u16, Random, 10000; + bench_dedup_u16_none_10000: |x| x as u16, None, 10000; + bench_dedup_u16_all_10000: |x| x as u16, All, 10000; + bench_dedup_u64_random_1000: |x| x as u64, Random, 1000; + bench_dedup_u64_none_1000: |x| x as u64, None, 1000; + bench_dedup_u64_all_1000: |x| x as u64, All, 1000; + bench_dedup_u64_random_10000: |x| x as u64, Random, 10000; + bench_dedup_u64_none_10000: |x| x as u64, None, 10000; + bench_dedup_u64_all_10000: |x| x as u64, All, 10000; + bench_dedup_wide_random_1000: dedup_gen_wide, Random, 1000; + bench_dedup_wide_none_1000: dedup_gen_wide, None, 1000; + bench_dedup_wide_all_1000: dedup_gen_wide, All, 1000; + bench_dedup_wide_random_10000: dedup_gen_wide, Random, 10000; + bench_dedup_wide_none_10000: dedup_gen_wide, None, 10000; + bench_dedup_wide_all_10000: dedup_gen_wide, All, 10000; } #[bench] diff --git a/library/alloctests/tests/bstr.rs b/library/alloctests/tests/byte_str.rs similarity index 99% rename from library/alloctests/tests/bstr.rs rename to library/alloctests/tests/byte_str.rs index 64a1b901f1300..f8484985d84fe 100644 --- a/library/alloctests/tests/bstr.rs +++ b/library/alloctests/tests/byte_str.rs @@ -1,4 +1,4 @@ -use alloc::bstr::ByteString; +use alloc::byte_str::ByteString; use core::assert_matches; #[test] diff --git a/library/alloctests/tests/lib.rs b/library/alloctests/tests/lib.rs index 5c8ee92388dd0..291ee9ddff3ff 100644 --- a/library/alloctests/tests/lib.rs +++ b/library/alloctests/tests/lib.rs @@ -8,9 +8,9 @@ #![feature(binary_heap_into_iter_sorted)] #![feature(binary_heap_pop_if)] #![feature(borrowed_buf_init)] -#![feature(bstr)] -#![feature(bstr_to_string)] #![feature(buf_read_has_data_left)] +#![feature(byte_str)] +#![feature(byte_str_to_string)] #![feature(can_vector)] #![feature(casefold)] #![feature(const_btree_len)] @@ -72,8 +72,8 @@ mod arc; mod autotraits; mod borrow; mod boxed; -mod bstr; mod btree_set_hash; +mod byte_str; mod c_str; mod c_str2; mod collections; diff --git a/library/alloctests/tests/sync.rs b/library/alloctests/tests/sync.rs index 5742855768c70..05f90a1a78cce 100644 --- a/library/alloctests/tests/sync.rs +++ b/library/alloctests/tests/sync.rs @@ -58,6 +58,7 @@ impl Drop for AllocCanary<'_> { #[test] #[cfg_attr(target_os = "emscripten", ignore)] +#[cfg_attr(target_os = "wasi", ignore)] fn manually_share_arc() { let v = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10]; let arc_v = Arc::new(v); @@ -454,6 +455,7 @@ fn test_ptr_eq() { #[test] #[cfg_attr(target_os = "emscripten", ignore)] +#[cfg_attr(target_os = "wasi", ignore)] fn test_weak_count_locked() { let mut a = Arc::new(atomic::AtomicBool::new(false)); let a2 = a.clone(); diff --git a/library/alloctests/tests/vec.rs b/library/alloctests/tests/vec.rs index 787452df7f680..cfb6a6bd03824 100644 --- a/library/alloctests/tests/vec.rs +++ b/library/alloctests/tests/vec.rs @@ -2534,6 +2534,94 @@ fn test_vec_dedup_panicking() { } } +#[test] +fn test_vec_dedup_long() { + fn check(template: &[T]) { + let mut expected = template.to_vec(); + let (kept, _) = expected.partition_dedup(); + let kept_len = kept.len(); + expected.truncate(kept_len); + + let mut vec = template.to_vec(); + vec.dedup(); + assert_eq!(vec, expected, "input: {template:?}"); + } + + macro_rules! check_type { + ($ty:ty) => { + for len in [0, 1, 2, 3, 4, 5, 7, 8, 9, 15, 16, 17, 31, 32, 33, 63, 64, 65] { + let distinct: Vec<$ty> = (0..len).map(|x| x as $ty).collect(); + check(&distinct); + check(&vec![7 as $ty; len]); + + for dup_at in 1..len { + let mut v = distinct.clone(); + v[dup_at] = v[dup_at - 1]; + check(&v); + + if dup_at + 1 < len { + let mut w = v.clone(); + w[len - 1] = w[len - 2]; + check(&w); + } + } + } + }; + } + + check_type!(u8); + check_type!(u16); + check_type!(u32); + check_type!(u64); +} + +#[test] +fn test_vec_dedup_long_boxed() { + for len in [9, 10, 16, 17, 24, 25, 33] { + for dup_at in 1..len { + let mut v: Vec> = (0..len as u32).map(Box::new).collect(); + v[dup_at] = Box::new(*v[dup_at - 1]); + let expected: Vec = (0..len as u32).filter(|&x| x != dup_at as u32).collect(); + v.dedup(); + let got: Vec = v.iter().map(|b| **b).collect(); + assert_eq!(got, expected, "len={len} dup_at={dup_at}"); + } + } +} + +#[test] +#[cfg_attr(not(panic = "unwind"), ignore = "test requires unwinding support")] +fn test_vec_dedup_eq_panics_for_simd_sized_type() { + static EQ_COUNT: AtomicU32 = AtomicU32::new(0); + static DROP_COUNT: AtomicU32 = AtomicU32::new(0); + + #[derive(Debug)] + #[repr(transparent)] + struct Panic(u32); + + impl PartialEq for Panic { + fn eq(&self, other: &Self) -> bool { + if EQ_COUNT.fetch_add(1, Ordering::SeqCst) == 2 { + panic!("comparison panic"); + } + self.0 == other.0 + } + } + + impl Drop for Panic { + fn drop(&mut self) { + DROP_COUNT.fetch_add(1, Ordering::SeqCst); + } + } + + EQ_COUNT.store(0, Ordering::SeqCst); + DROP_COUNT.store(0, Ordering::SeqCst); + let mut vec: Vec<_> = [0, 0, 1, 2, 3, 4, 5, 6, 7].into_iter().map(Panic).collect(); + assert!(catch_unwind(AssertUnwindSafe(|| vec.dedup())).is_err()); + drop(vec); + assert_eq!(DROP_COUNT.load(Ordering::SeqCst), 9); +} + // Regression test for issue #82533 #[test] #[cfg_attr(not(panic = "unwind"), ignore = "test requires unwinding support")] diff --git a/library/compiler-builtins/.editorconfig b/library/compiler-builtins/.editorconfig index 23a4d23f28200..a7805a4df7b1d 100644 --- a/library/compiler-builtins/.editorconfig +++ b/library/compiler-builtins/.editorconfig @@ -12,3 +12,6 @@ indent_size = 4 [*.{yaml,yml}] indent_size = 2 + +[*.json*] +indent_size = 2 diff --git a/library/compiler-builtins/.github/renovate.json5 b/library/compiler-builtins/.github/renovate.json5 index 7d18fd573d6d8..e1656528ed68a 100644 --- a/library/compiler-builtins/.github/renovate.json5 +++ b/library/compiler-builtins/.github/renovate.json5 @@ -5,26 +5,37 @@ ":maintainLockFilesMonthly", "helpers:pinGitHubActionDigestsToSemver" ], + commitMessagePrefix: "deps:", + customManagers: [ + { + customType: "regex", + managerFilePatterns: [".github/workflows/main.yaml"], + matchStrings: ["BENCHMARK_RUSTC:\\s*(?\\S+)"], + autoReplaceStringTemplate: "BENCHMARK_RUSTC: {{{newValue}}}", + depNameTemplate: "nightly", + datasourceTemplate: "rust-version", + versioningTemplate: "rust-release-channel", + }, + ], packageRules: [ { - matchCategories: [ - "rust" - ], - matchJsonata: [ - "isBreaking != true" - ], + matchCategories: ["rust"], + matchJsonata: ["isBreaking != true"], // Disable non-breaking change updates because they // are updated periodically with lockfile maintainance. enabled: false, }, { - matchManagers: [ - "github-actions" - ], + matchManagers: ["github-actions"], // Every month schedule: "* 0 1 * *", + commitMessagePrefix: "ci:", groupName: "Github Actions", - } + }, + { + matchManagers: ["custom.regex"], + commitMessage: "bench: Update pinned toolchain to {{{newVersion}}}", + }, ], // Receive any update that fixes security vulnerabilities. // We need this because we disabled "patch" updates for Rust. diff --git a/library/compiler-builtins/.github/workflows/main.yaml b/library/compiler-builtins/.github/workflows/main.yaml index 53e712528aa63..1acd01725f15f 100644 --- a/library/compiler-builtins/.github/workflows/main.yaml +++ b/library/compiler-builtins/.github/workflows/main.yaml @@ -16,7 +16,7 @@ env: RUSTDOCFLAGS: -Dwarnings RUSTFLAGS: -Dwarnings -Dlinker_messages RUST_BACKTRACE: full - BENCHMARK_RUSTC: nightly-2026-08-19 # Pin the toolchain for reproducable results + BENCHMARK_RUSTC: nightly-2026-09-26 # Pin the toolchain for reproducable results defaults: run: @@ -51,7 +51,7 @@ jobs: test: name: Build and test - timeout-minutes: 60 + timeout-minutes: 90 # NOTE: self-hosted riscv64 runners are experimental and may be flaky. # Do not block CI on failures from this platform for now. continue-on-error: ${{ contains(matrix.os, 'self-hosted') }} @@ -71,6 +71,8 @@ jobs: os: ubuntu-26.04 - target: armv7-unknown-linux-gnueabihf os: ubuntu-26.04 + - target: armv7-unknown-linux-gnueabihf + os: self-hosted-linux-armhf-noble-large-rust - target: thumbv7neon-unknown-linux-gnueabihf os: ubuntu-26.04 - target: i586-unknown-linux-gnu @@ -94,8 +96,8 @@ jobs: # os: ["self-hosted", "linux", "riscv64"] - target: riscv64gc-unknown-linux-gnu os: ubuntu-26.04 - - target: s390x-unknown-linux-gnu - os: self-hosted-linux-s390x-resolute-large-rust # resolute == ubuntu-26.04 + # - target: s390x-unknown-linux-gnu + # os: self-hosted-linux-s390x-resolute-large-rust # resolute == ubuntu-26.04 - target: s390x-unknown-linux-gnu os: ubuntu-24.04-s390x - target: thumbv6m-none-eabi @@ -149,7 +151,7 @@ jobs: - name: Set up dependencies and Rust run: ./ci/install-test-deps.sh "$JOB_TARGET" "$JOB_HOST_TOOLCHAIN" - - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 + - uses: taiki-e/install-action@f7e5d7c961414b23f5b25b2da9294395d08513ad # v2.87.26 continue-on-error: true with: tool: nextest@0.9.131 @@ -169,7 +171,7 @@ jobs: key: ${{ matrix.target }}-buildx-${{ github.sha }} restore-keys: ${{ matrix.target }}-buildx- # Configure buildx to use Docker layer caching - - uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0 + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 if: ${{ env.RUN_IN_DOCKER == 'true' }} - name: Cache compiler-rt @@ -243,7 +245,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: { persist-credentials: false } - - uses: zizmorcore/zizmor-action@3dc1ecc9bcb9e94e9b2c709687979e1298497054 # v0.6.2 + - uses: zizmorcore/zizmor-action@cc914d7f3750a2d13d75c7f184a1060aa0e9d482 # v0.6.4 build-custom: name: Build custom target @@ -305,7 +307,7 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: { persist-credentials: false } - - uses: taiki-e/install-action@7f4eb899022d8fe70b20c4f3de697aa85c309026 # v2.85.11 + - uses: taiki-e/install-action@f7e5d7c961414b23f5b25b2da9294395d08513ad # v2.87.26 with: tool: cargo-binstall@1.17.7 @@ -324,7 +326,7 @@ jobs: run: ./ci/bench-icount.sh "$JOB_TARGET" - name: Upload the benchmark baseline - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: ${{ env.BASELINE_NAME }} path: ${{ env.BASELINE_NAME }}.tar.xz diff --git a/library/compiler-builtins/.github/workflows/publish.yaml b/library/compiler-builtins/.github/workflows/publish.yaml index 02ddf0babdf40..04daff8900959 100644 --- a/library/compiler-builtins/.github/workflows/publish.yaml +++ b/library/compiler-builtins/.github/workflows/publish.yaml @@ -3,6 +3,7 @@ name: Release-plz permissions: pull-requests: write contents: write + id-token: write on: push: { branches: [main] } @@ -10,7 +11,7 @@ on: jobs: release-plz: name: Release-plz - runs-on: ubuntu-24.04 + runs-on: ubuntu-26.04 environment: publish steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -20,6 +21,6 @@ jobs: - name: Install Rust (rustup) run: rustup update nightly --no-self-update && rustup default nightly - name: Run release-plz - uses: release-plz/action@2eb1d8bcb770b4c48ccfaad919734b38b51958c9 # v0.5 + uses: release-plz/action@18641b6c63063cc5ff8786ebf69cd57e8541bae1 # v0.5.140 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/library/compiler-builtins/.github/workflows/rustc-pull.yml b/library/compiler-builtins/.github/workflows/rustc-pull.yml index 34e78e445677c..04238956fd33c 100644 --- a/library/compiler-builtins/.github/workflows/rustc-pull.yml +++ b/library/compiler-builtins/.github/workflows/rustc-pull.yml @@ -13,7 +13,8 @@ env: jobs: pull: if: github.repository == 'rust-lang/compiler-builtins' - uses: rust-lang/josh-sync/.github/workflows/rustc-pull.yml@00d007cf377c524bc2968b40a61ef1d2ce20e283 # main + # zizmor: ignore[unpinned-uses] We explicitly want to track the main branch. + uses: rust-lang/josh-sync/.github/workflows/rustc-pull.yml@b1da651cd99a7325cc242548db6d711639b0c30c # main with: github-app-id: ${{ vars.APP_CLIENT_ID }} pr-author: "workflows-compiler-builtins[bot]" diff --git a/library/compiler-builtins/Cargo.lock b/library/compiler-builtins/Cargo.lock index e203f4d4cfbdd..ed159e70b5dd7 100644 --- a/library/compiler-builtins/Cargo.lock +++ b/library/compiler-builtins/Cargo.lock @@ -133,9 +133,9 @@ dependencies = [ [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "bstr" @@ -174,9 +174,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.4.4" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" dependencies = [ "find-msvc-tools", "shlex", @@ -184,9 +184,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "chacha20" @@ -228,9 +228,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -238,9 +238,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstream", "anstyle", @@ -250,21 +250,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.4" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] name = "clap_lex" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "colorchoice" @@ -281,9 +281,9 @@ dependencies = [ [[package]] name = "console" -version = "0.16.4" +version = "0.16.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe5f465a4f6fee88fad41b85d990f84c835335e85b5d9e6e63e0d06d28cba7c" +checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3" dependencies = [ "encode_unicode", "libc", @@ -301,9 +301,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" dependencies = [ "cfg-if", ] @@ -344,9 +344,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -354,18 +354,18 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.20" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crunchy" @@ -408,9 +408,9 @@ checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" [[package]] name = "either" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" [[package]] name = "either-or-both" @@ -446,9 +446,9 @@ dependencies = [ [[package]] name = "escape8259" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5692dd7b5a1978a5aeb0ce83b7655c58ca8efdcb79d21036ea249da95afec2c6" +checksum = "9f1f0d310082fd5de539d4f41601182241147498444a7695c520288a0cdbead6" [[package]] name = "fastrand" @@ -458,9 +458,9 @@ checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" [[package]] name = "find-msvc-tools" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "flate2" @@ -538,21 +538,28 @@ dependencies = [ [[package]] name = "gungraun" -version = "0.19.4" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9300d38d30facf6870c15bc9e243edcc47e5ddc270c9d2c1ae824a00b1faea79" +checksum = "c041375e48b8ef7cd2d0e70715598154af627c1a551e41c35fdb4a2b3910831a" dependencies = [ "bincode-next", "derive_more", + "gungraun-common", "gungraun-macros", "gungraun-runner", ] +[[package]] +name = "gungraun-common" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc270ba55639babb5806b62d4bfbc35e15ca15791e3ec00c7b92b39c62a9d8eb" + [[package]] name = "gungraun-macros" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7f3214bae6cfd6739f4f29edb262bae439c6393a7b7b12c4eeb96417c2e50df" +checksum = "08472a9e996a98a5b4274dd342fb32ebe1a5cfeb2662b6578d0c7bf1dd203129" dependencies = [ "derive_more", "proc-macro-error3", @@ -561,14 +568,14 @@ dependencies = [ "rustc_version", "serde", "serde_json", - "syn 2.0.119", + "syn 3.0.6", ] [[package]] name = "gungraun-runner" -version = "0.19.4" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47c68c92812da579a2510a2cc702d5aa3251992b9662d4a98502ca46e0cbe16e" +checksum = "1e1460377fe936c3efbe8e27c422b745fa54ebaea81ffe4a53a6f17c7a478683" dependencies = [ "either-or-both", "serde", @@ -599,9 +606,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "indexmap" -version = "2.14.1" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown", @@ -642,9 +649,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "js-sys" -version = "0.3.104" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +checksum = "7883d941dae510fb2d978fc3fe018c71c9e2892fd38854de3e8b92c2e5ad9cc5" dependencies = [ "cfg-if", "futures-util", @@ -653,9 +660,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.189" +version = "0.2.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" [[package]] name = "libm" @@ -678,7 +685,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -754,7 +761,7 @@ checksum = "fc80370b4544f28ffa317e3c3474ee3ecbfe269196c01ae657d9f837a7d944a1" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -927,7 +934,7 @@ dependencies = [ "proc-macro-error-attr3", "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -956,9 +963,9 @@ checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" [[package]] name = "rand" -version = "0.10.2" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +checksum = "65c9fb96cbc91e3478eaae79a69fcd3f1ae4ad052e471fe6732fff548984b4af" dependencies = [ "chacha20", "getrandom", @@ -1081,9 +1088,9 @@ dependencies = [ [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" dependencies = [ "bitflags", "errno", @@ -1149,7 +1156,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] @@ -1185,9 +1192,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" [[package]] name = "strsim" @@ -1221,15 +1228,19 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "sysbench" +version = "0.1.0" + [[package]] name = "tempfile" version = "3.27.0" @@ -1261,15 +1272,15 @@ dependencies = [ [[package]] name = "twox-hash" -version = "2.1.4" +version = "2.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5283634e518fe9e82c7b20520bb4bc209009fd16c82077c802f8111ecbb0117a" +checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" [[package]] name = "unicode-width" @@ -1338,9 +1349,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +checksum = "9bb54f33acc68fd454578d9820b0bde1a1a3d17aa17bb7b6595806d02886d409" dependencies = [ "cfg-if", "once_cell", @@ -1351,9 +1362,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +checksum = "2e29d0c35b16e224a7eeb5cd2d25e3e1968fbd65604117b44d3b789d00ee8535" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1361,22 +1372,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +checksum = "6f501a8bc3719dba86ef8ae4728879c08001bea749eb1333ac5b91e040e2a6b7" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.6", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.127" +version = "0.2.129" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +checksum = "23f0c9c52aa7cd7d77769a4cfe2a9adb1b331f489a41d912ce14513d5ab995c6" dependencies = [ "unicode-ident", ] @@ -1392,9 +1403,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.104" +version = "0.3.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +checksum = "88261b9deccee56594c11a3460c462c41f58d148598fe70ad77070126a68aba4" dependencies = [ "js-sys", "wasm-bindgen", @@ -1464,18 +1475,18 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "zerocopy" -version = "0.8.56" +version = "0.8.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +checksum = "86502bf56ac7c77571a32e2647bb2a15894565e981fb2a48d7bde2d91c965a9d" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.56" +version = "0.8.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +checksum = "5457206954b06561e2608c7e19cf58b1926586d999c246eebe4502f7e2039d1a" dependencies = [ "proc-macro2", "quote", @@ -1484,9 +1495,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.6.7" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" [[package]] name = "zmij" diff --git a/library/compiler-builtins/Cargo.toml b/library/compiler-builtins/Cargo.toml index 423adfbcbd02c..2091b3aa0f2fc 100644 --- a/library/compiler-builtins/Cargo.toml +++ b/library/compiler-builtins/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/musl-math-sys", "crates/panic-handler", "crates/symcheck", + "crates/sysbench", "crates/update-api-list", "crates/util", "libm", @@ -45,7 +46,7 @@ getopts = "0.2.24" getrandom = "0.4.1" glob = "0.3.3" gmp-mpfr-sys = { version = "1.6.8", default-features = false } -gungraun = "0.19.0" +gungraun = "0.20.0" heck = "0.5.0" indicatif = { version = "0.18.3", default-features = false } libm = { path = "libm", default-features = false } diff --git a/library/compiler-builtins/builtins-test-intrinsics/Cargo.lock b/library/compiler-builtins/builtins-test-intrinsics/Cargo.lock index 4461cbc35bcc8..73f4f96db2d20 100644 --- a/library/compiler-builtins/builtins-test-intrinsics/Cargo.lock +++ b/library/compiler-builtins/builtins-test-intrinsics/Cargo.lock @@ -15,9 +15,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.4" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +checksum = "f74872d07caf508b30a21f6836e7d7016a2eaf7d9ff4f48deaa58cd8a0407630" dependencies = [ "find-msvc-tools", "shlex", @@ -32,9 +32,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.11" +version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484" [[package]] name = "panic-handler" diff --git a/library/compiler-builtins/builtins-test-intrinsics/src/main.rs b/library/compiler-builtins/builtins-test-intrinsics/src/main.rs index 13bc7337ba3ec..7673e1dbe0bac 100644 --- a/library/compiler-builtins/builtins-test-intrinsics/src/main.rs +++ b/library/compiler-builtins/builtins-test-intrinsics/src/main.rs @@ -10,7 +10,6 @@ #![feature(f128)] #![feature(f16)] #![feature(lang_items)] -#![feature(optimize_attribute)] #![no_std] #![no_main] diff --git a/library/compiler-builtins/builtins-test/benches/float_pow.rs b/library/compiler-builtins/builtins-test/benches/float_pow.rs index d749ae47ddafb..3d6841525b013 100644 --- a/library/compiler-builtins/builtins-test/benches/float_pow.rs +++ b/library/compiler-builtins/builtins-test/benches/float_pow.rs @@ -22,9 +22,7 @@ float_bench! { asm: [], } -// FIXME(f16_f128): can be changed to only `f128_enabled` once `__multf3` and `__divtf3` are -// distributed by nightly. -#[cfg(all(f128_enabled, not(no_sys_f128)))] +#[cfg(f128_enabled)] float_bench! { name: powi_f128, sig: (a: f128, b: i32) -> f128, @@ -42,7 +40,7 @@ pub fn float_pow() { powi_f32(&mut criterion); powi_f64(&mut criterion); - #[cfg(all(f128_enabled, not(no_sys_f128)))] + #[cfg(f128_enabled)] powi_f128(&mut criterion); } diff --git a/library/compiler-builtins/builtins-test/tests/conv.rs b/library/compiler-builtins/builtins-test/tests/conv.rs index ac808b0ae45c7..d730c2f393f16 100644 --- a/library/compiler-builtins/builtins-test/tests/conv.rs +++ b/library/compiler-builtins/builtins-test/tests/conv.rs @@ -41,13 +41,37 @@ mod i_to_f { ); let f1: $f_ty = $fn(x); + if f1.is_nan() { + panic!("int->float cannot result in a NaN: {}({x})", stringify!($fn)); + } + #[cfg($sys_available)] { + // If `f1` is infinite, calculate the integer that is 1 ULP after/before + // $f_ty::MAX/$f_ty::MIN, saturating as needed. There's no need to + // handle NaN as the result of a conversion can never be NaN. + let as_int = |f: $f_ty| { + if f.is_infinite() { + // For infinity, calculate the float that is half of the desired + // 1-ulp-after-max/-before-min (and therefore representable as + // $f_ty), then multiply by two after converting to an integer. + let bits = <$f_ty>::EXP_MASK - <$f_ty>::IMPLICIT_BIT; + let bits = if f.is_sign_negative() { + bits | <$f_ty as Float>::SIGN_MASK + } else { + bits + }; + // float->int casts saturate, so saturate the multiply to match. + (<$f_ty>::from_bits(bits) as $i_ty).saturating_mul(2) + } else { + f as $i_ty + } + }; // This makes sure that the conversion produced the best rounding possible, and does // this independent of `x as $into` rounding correctly. // This assumes that float to integer conversion is correct. - let y_minus_ulp = <$f_ty>::from_bits(f1.to_bits().wrapping_sub(1)) as $i_ty; - let y = f1 as $i_ty; - let y_plus_ulp = <$f_ty>::from_bits(f1.to_bits().wrapping_add(1)) as $i_ty; + let y_minus_ulp = as_int(f1.next_down()); + let y = as_int(f1); + let y_plus_ulp = as_int(f1.next_up()); let error_minus = <$i_ty as Int>::abs_diff(y_minus_ulp, x); let error = <$i_ty as Int>::abs_diff(y, x); let error_plus = <$i_ty as Int>::abs_diff(y_plus_ulp, x); diff --git a/library/compiler-builtins/builtins-test/tests/lse.rs b/library/compiler-builtins/builtins-test/tests/lse.rs index 20d4d19a328a1..463d8a8d9c238 100644 --- a/library/compiler-builtins/builtins-test/tests/lse.rs +++ b/library/compiler-builtins/builtins-test/tests/lse.rs @@ -58,7 +58,7 @@ mod cas { crate::run_fuzz_tests_with_lse_variants(10000, |expected: super::int_ty!($bytes), new| { let mut target = expected.wrapping_add(10); let ret: super::int_ty!($bytes) = unsafe { - compiler_builtins::aarch64_outline_atomics::$name::$name( + compiler_builtins::aarch64_outline_atomics::$name( expected, new, &mut target, @@ -77,7 +77,7 @@ mod cas { target = expected; let ret: super::int_ty!($bytes) = unsafe { - compiler_builtins::aarch64_outline_atomics::$name::$name( + compiler_builtins::aarch64_outline_atomics::$name( expected, new, &mut target, @@ -108,9 +108,7 @@ mod swap { let orig_right = right; assert_eq!( unsafe { - compiler_builtins::aarch64_outline_atomics::$name::$name( - left, &mut right, - ) + compiler_builtins::aarch64_outline_atomics::$name(left, &mut right) }, orig_right ); @@ -132,7 +130,13 @@ macro_rules! test_op { let mut target = old; let op: fn(super::int_ty!($bytes), super::int_ty!($bytes)) -> _ = $($op)*; let expected = op(old, val); - assert_eq!(old, unsafe { compiler_builtins::aarch64_outline_atomics::$name::$name(val, &mut target) }, "{} should return original value", stringify!($name)); + assert_eq!( + old, + unsafe { + compiler_builtins::aarch64_outline_atomics::$name(val, &mut target) + }, + "{} should return original value", stringify!($name) + ); assert_eq!(expected, target, "{} should store to target", stringify!($name)); }); } diff --git a/library/compiler-builtins/builtins-test/tests/mem.rs b/library/compiler-builtins/builtins-test/tests/mem.rs index f7ae9c9b5c3a4..bafee5807e10a 100644 --- a/library/compiler-builtins/builtins-test/tests/mem.rs +++ b/library/compiler-builtins/builtins-test/tests/mem.rs @@ -1,5 +1,3 @@ -#![feature(optimize_attribute)] - use compiler_builtins::mem::{memcmp, memcpy, memmove, memset, strlen, wchar_t}; const WORD_SIZE: usize = core::mem::size_of::(); diff --git a/library/compiler-builtins/ci/install-test-deps.sh b/library/compiler-builtins/ci/install-test-deps.sh index b64bcf654add0..cd197a7d915e9 100755 --- a/library/compiler-builtins/ci/install-test-deps.sh +++ b/library/compiler-builtins/ci/install-test-deps.sh @@ -21,12 +21,34 @@ fi to_install=() if [ "$RUN_IN_DOCKER" != "0" ]; then - ! command -v rustup && to_install+=(rustup) - ! command -v m4 && to_install+=(m4) + command -v libtool || to_install+=(libtool) + command -v m4 || to_install+=(m4) + command -v make || to_install+=(make) + command -v rustup || to_install+=(rustup) fi if [ "$target" = "i686-pc-windows-gnu" ]; then - ! command -v i686-w64-mingw32-gcc && to_install+=(mingw-w64-i686-gcc) + command -v i686-w64-mingw32-gcc || to_install+=(mingw-w64-i686-gcc) +fi + +# specific check for armv7-unknown-linux-gnueabihf on canonical runners +# which are 32-bit armhf-native but have VMs running over an arm64 kernel with AArch32/EL0 support +if [ "$target" = "armv7-unknown-linux-gnueabihf" ] && [ "${RUN_IN_DOCKER:-}" != "true" ] && + [ "$(dpkg --print-architecture 2>/dev/null)" = "arm64" ]; then + + # expose these variables so we can keep the arm64 host toolchain, cross-link with + # the armhf gcc, and run the test binaries natively in AArch32 mode + { + echo "CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc" + echo "CC_armv7_unknown_linux_gnueabihf=arm-linux-gnueabihf-gcc" + echo "AR_armv7_unknown_linux_gnueabihf=arm-linux-gnueabihf-ar" + } >> "$GITHUB_ENV" + + # install required tooling to enable cross-compilation + sudo dpkg --add-architecture armhf + to_install+=(gcc-arm-linux-gnueabihf) # cross-compiler + to_install+=(libc6-dev-armhf-cross) # target libc + to_install+=(libc6:armhf libgcc-s1:armhf) # loaded at runtime fi if [ ${#to_install[@]} -ne 0 ]; then @@ -43,6 +65,7 @@ if [ ${#to_install[@]} -ne 0 ]; then fi # Install the correct Rust version +rustup set profile minimal rustup update "$channel" --no-self-update rustup default "$channel" rustup target add "$target" diff --git a/library/compiler-builtins/ci/run.sh b/library/compiler-builtins/ci/run.sh index 7abc44e1c2b6a..8167158ef49c9 100755 --- a/library/compiler-builtins/ci/run.sh +++ b/library/compiler-builtins/ci/run.sh @@ -6,11 +6,11 @@ export RUST_BACKTRACE="${RUST_BACKTRACE:-full}" export NEXTEST_STATUS_LEVEL=all target="${1:-}" +host_tuple="$(rustc --print host-tuple)" if [ -z "$target" ]; then - host_target=$(rustc -vV | awk '/^host/ { print $2 }') - echo "Defaulted to host target $host_target" - target="$host_target" + echo "Defaulted to host target $host_tuple" + target="$host_tuple" fi # Print machine information @@ -58,6 +58,9 @@ fi if [ "${BUILD_ONLY:-}" = "1" ]; then echo "no tests to run for build-only targets" else + # Get a quick performance score + cargo run -p sysbench --target "$target" --release + test_builtins=( "${test_runner[@]}" --package builtins-test @@ -138,7 +141,7 @@ echo "::endgroup" # Test libm # Make sure a simple build works -cargo check -p libm --no-default-features --target "$target" +asgroup cargo check -p libm --no-default-features --target "$target" if [ "${MAY_SKIP_LIBM_CI:-}" = "true" ]; then echo "skipping libm PR CI" @@ -171,19 +174,21 @@ case "$target" in *) mflags+=(--features libm-test/build-musl) ;; esac - # Configure which targets test against MPFR case "$target" in - # MSVC cannot link MPFR + # Excluded in gmp-mpfr-sys with "Windows MSVC target is not supported (linking + # would fail)". MPFR isn't really built for non-gcc-like compilers anyway. *windows-msvc*) ;; # FIXME: MinGW should be able to build MPFR, but setup in CI is nontrivial. *windows-gnu*) ;; - # Targets that aren't cross compiled in CI work fine + # Ensure enabled on targets that we expect to be native in CI aarch64*apple*) mflags+=(--features libm-test/build-mpfr) ;; aarch64*linux*) mflags+=(--features libm-test/build-mpfr) ;; i586*) mflags+=(--features libm-test/build-mpfr --features gmp-mpfr-sys/force-cross) ;; i686*) mflags+=(--features libm-test/build-mpfr) ;; x86_64*) mflags+=(--features libm-test/build-mpfr) ;; + # Fallback to enabled on native targets, disabled otherwise + *) [ "$target" = "$host_tuple" ] && mflags+=(--features libm-test/build-mpfr) ;; esac # FIXME: `STATUS_DLL_NOT_FOUND` testing macros on CI. diff --git a/library/compiler-builtins/compiler-builtins/src/aarch64_outline_atomics.rs b/library/compiler-builtins/compiler-builtins/src/aarch64_outline_atomics.rs index 367768ecb60d6..f51d076317516 100644 --- a/library/compiler-builtins/compiler-builtins/src/aarch64_outline_atomics.rs +++ b/library/compiler-builtins/compiler-builtins/src/aarch64_outline_atomics.rs @@ -148,77 +148,6 @@ macro_rules! stxp { }; } -// The AArch64 assembly syntax for relocation specifiers -// when accessing symbols changes depending on the target executable format. -// In ELF (used in Linux), we have a prefix notation surrounded by colons (:specifier:sym), -// while in Mach-O object files (used in MacOS), a postfix notation is used (sym@specifier). - -/// AArch64 ELF position-independent addressing: -/// -/// adrp xN, symbol -/// add xN, xN, :lo12:symbol -/// -/// The :lo12: modifier selects the low 12 bits of the symbol address -/// and emits an ELF relocation such as R_AARCH64_ADD_ABS_LO12_NC. -/// -/// Defined by the AArch64 ELF psABI. -/// See: . -#[cfg(not(target_vendor = "apple"))] -macro_rules! sym { - ($sym:literal) => { - $sym - }; -} - -#[cfg(not(target_vendor = "apple"))] -macro_rules! sym_off { - ($sym:literal) => { - concat!(":lo12:", $sym) - }; -} - -/// Mach-O ARM64 relocation types: -/// ARM64_RELOC_PAGE21 -/// ARM64_RELOC_PAGEOFF12 -/// -/// These relocations implement the @PAGE / @PAGEOFF split used by -/// adrp + add sequences on Apple platforms. -/// -/// adrp xN, symbol@PAGE -> ARM64_RELOC_PAGE21 -/// add xN, xN, symbol@PAGEOFF -> ARM64_RELOC_PAGEOFF12 -/// -/// Relocation types defined by Apple in XNU: . -/// See: . -#[cfg(target_vendor = "apple")] -macro_rules! sym { - ($sym:literal) => { - concat!($sym, "@PAGE") - }; -} - -#[cfg(target_vendor = "apple")] -macro_rules! sym_off { - ($sym:literal) => { - concat!($sym, "@PAGEOFF") - }; -} - -// If supported, perform the requested LSE op and return, or fallthrough. -macro_rules! try_lse_op { - ($op: literal, $ordering:ident, $bytes:tt, $($reg:literal,)* [ $mem:ident ] ) => { - concat!( - ".arch_extension lse\n", - concat!("adrp x16, ", sym!("{have_lse}"), "\n"), - concat!("ldrb w16, [x16, ", sym_off!("{have_lse}"), "]\n"), - "cbz w16, 8f\n", - // LSE_OP s(reg),* [$mem] - concat!(lse!($op, $ordering, $bytes), $( " ", reg!($bytes, $reg), ", " ,)* "[", stringify!($mem), "]\n",), - "ret - 8:" - ) - }; -} - // Translate memory ordering to the LSE suffix #[rustfmt::skip] macro_rules! lse_mem_sfx { @@ -238,34 +167,72 @@ macro_rules! lse { }; } +// Ensure the register holding the pointer is zero-extended on ilp32 targets +// before dereferencing it. This may not work, see llvm/llvm-project#225834. +// +// This write requires the pointer argument to be made `inout` in the asm! +// sections. If the ilp32 targets are removed, this macro should be removed +// and the pointer argument changed to `in`. +#[cfg(target_pointer_width = "32")] +macro_rules! sanitize_ptr { + ($num:literal) => { + concat!("mov w", $num, ", w", $num) + }; +} + +#[cfg(not(target_pointer_width = "32"))] +macro_rules! sanitize_ptr { + ($num:literal) => { + "" + }; +} + /// See . macro_rules! compare_and_swap { ($ordering:ident, $bytes:tt, $name:ident) => { intrinsics! { #[maybe_use_optimized_c_shim] - #[unsafe(naked)] pub unsafe extern "C" fn $name ( expected: int_ty!($bytes), desired: int_ty!($bytes), ptr: *mut int_ty!($bytes) ) -> int_ty!($bytes) { - // We can't use `AtomicI8::compare_and_swap`; we *are* compare_and_swap. - core::arch::naked_asm! { - // CAS s(0), s(1), [x2]; if LSE supported. - try_lse_op!("cas", $ordering, $bytes, 0, 1, [x2]), - // UXT s(tmp0), s(0) - concat!(uxt!($bytes), " ", reg!($bytes, 16), ", ", reg!($bytes, 0)), - "0:", - // LDXR s(0), [x2] - concat!(ldxr!($ordering, $bytes), " ", reg!($bytes, 0), ", [x2]"), - // cmp s(0), s(tmp0) - concat!("cmp ", reg!($bytes, 0), ", ", reg!($bytes, 16)), - "bne 1f", - // STXR w(tmp1), s(1), [x2] - concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 1), ", [x2]"), - "cbnz w17, 0b", - "1:", - "ret", - have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, + let mut expected = expected; + unsafe { + if HAVE_LSE_ATOMICS.load(Ordering::Relaxed) != 0 { + core::arch::asm!( + ".arch_extension lse", + sanitize_ptr!(2), + // CAS s(0), s(1), [x2]; if LSE supported. + concat!(lse!("cas", $ordering, $bytes), " ", reg!($bytes, 0), ", ", reg!($bytes, 1),", [x2]"), + inlateout("x0") expected, + in("x1") desired, + inout("x2") ptr => _, + options(nostack, preserves_flags), + ); + } else { + core::arch::asm!( + sanitize_ptr!(2), + // UXT s(tmp0), s(0) + concat!(uxt!($bytes), " ", reg!($bytes, 16), ", ", reg!($bytes, 0)), + "1:", + // LDXR s(0), [x2] + concat!(ldxr!($ordering, $bytes), " ", reg!($bytes, 0), ", [x2]"), + // cmp s(0), s(tmp0) + concat!("cmp ", reg!($bytes, 0), ", ", reg!($bytes, 16)), + "bne 2f", + // STXR w(tmp1), s(1), [x2] + concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 1), ", [x2]"), + "cbnz w17, 1b", + "2:", + inlateout("x0") expected, + in("x1") desired, + inout("x2") ptr => _, + out("x16") _, + out("w17") _, + options(nostack), + ); + } } + expected } } }; @@ -276,28 +243,56 @@ macro_rules! compare_and_swap_u128 { ($ordering:ident, $name:ident) => { intrinsics! { #[maybe_use_optimized_c_shim] - #[unsafe(naked)] pub unsafe extern "C" fn $name ( expected: u128, desired: u128, ptr: *mut u128 ) -> u128 { - core::arch::naked_asm! { - // CASP x0, x1, x2, x3, [x4]; if LSE supported. - try_lse_op!("cas", $ordering, 16, 0, 1, 2, 3, [x4]), - "mov x16, x0", - "mov x17, x1", - "0:", - // LDXP x0, x1, [x4] - concat!(ldxp!($ordering), " x0, x1, [x4]"), - "cmp x0, x16", - "ccmp x1, x17, #0, eq", - "bne 1f", - // STXP w(tmp2), x2, x3, [x4] - concat!(stxp!($ordering), " w15, x2, x3, [x4]"), - "cbnz w15, 0b", - "1:", - "ret", - have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, + let mut expected_lo = expected as u64; + let mut expected_hi = (expected >> 64) as u64; + let desired_lo = desired as u64; + let desired_hi = (desired >> 64) as u64; + + unsafe { + if HAVE_LSE_ATOMICS.load(Ordering::Relaxed) != 0 { + core::arch::asm!( + ".arch_extension lse", + sanitize_ptr!(4), + // CASP x0, x1, x2, x3, [x4]; if LSE supported. + concat!(lse!("cas", $ordering, 16), " x0, x1, x2, x3, [x4]"), + inlateout("x0") expected_lo, + inlateout("x1") expected_hi, + in("x2") desired_lo, + in("x3") desired_hi, + inout("x4") ptr => _, + options(nostack, preserves_flags), + ); + } else { + core::arch::asm!( + sanitize_ptr!(4), + "mov x16, x0", + "mov x17, x1", + "1:", + // LDXP x0, x1, [x4] + concat!(ldxp!($ordering), " x0, x1, [x4]"), + "cmp x0, x16", + "ccmp x1, x17, #0x0, eq", + "b.ne 2f", + // STXP w(tmp2), x2, x3, [x4] + concat!(stxp!($ordering), " w15, x2, x3, [x4]"), + "cbnz w15, 1b", + "2:", + inlateout("x0") expected_lo, + inlateout("x1") expected_hi, + in("x2") desired_lo, + in("x3") desired_hi, + inout("x4") ptr => _, + out("w15") _, + out("x16") _, + out("x17") _, + options(nostack), + ); + } } + return ((expected_hi as u128) << 64) | expected_lo as u128; } } }; @@ -308,24 +303,40 @@ macro_rules! swap { ($ordering:ident, $bytes:tt, $name:ident) => { intrinsics! { #[maybe_use_optimized_c_shim] - #[unsafe(naked)] pub unsafe extern "C" fn $name ( left: int_ty!($bytes), right_ptr: *mut int_ty!($bytes) ) -> int_ty!($bytes) { - core::arch::naked_asm! { - // SWP s(0), s(0), [x1]; if LSE supported. - try_lse_op!("swp", $ordering, $bytes, 0, 0, [x1]), - // mov s(tmp0), s(0) - concat!("mov ", reg!($bytes, 16), ", ", reg!($bytes, 0)), - "0:", - // LDXR s(0), [x1] - concat!(ldxr!($ordering, $bytes), " ", reg!($bytes, 0), ", [x1]"), - // STXR w(tmp1), s(tmp0), [x1] - concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 16), ", [x1]"), - "cbnz w17, 0b", - "ret", - have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, + let mut left = left; + unsafe { + if HAVE_LSE_ATOMICS.load(Ordering::Relaxed) != 0 { + core::arch::asm! { + ".arch_extension lse", + sanitize_ptr!(1), + // SWP s(0), s(0), [x1]; if LSE supported. + concat!(lse!("swp", $ordering, $bytes), " ", reg!($bytes, 0), ", ", reg!($bytes, 0), ", [x1]"), + inlateout("x0") left, + inout("x1") right_ptr => _, + options(nostack, preserves_flags), + }; + } else { + core::arch::asm! { + sanitize_ptr!(1), + concat!("mov ", reg!($bytes, 16), ", ", reg!($bytes, 0)), + "1:", + // LDXR s(0), [x1] + concat!(ldxr!($ordering, $bytes), " ", reg!($bytes, 0), ", [x1]"), + // STXR w(tmp1), s(tmp0), [x1] + concat!(stxr!($ordering, $bytes), " w17, ", reg!($bytes, 16), ", [x1]"), + "cbnz w17, 1b", + inlateout("x0") left, + inout("x1") right_ptr => _, + out("x16") _, + out("w17") _, + options(nostack, preserves_flags), + }; + } } + left } } }; @@ -336,26 +347,44 @@ macro_rules! fetch_op { ($ordering:ident, $bytes:tt, $name:ident, $op:literal, $lse_op:literal) => { intrinsics! { #[maybe_use_optimized_c_shim] - #[unsafe(naked)] pub unsafe extern "C" fn $name ( val: int_ty!($bytes), ptr: *mut int_ty!($bytes) ) -> int_ty!($bytes) { - core::arch::naked_asm! { - // LSEOP s(0), s(0), [x1]; if LSE supported. - try_lse_op!($lse_op, $ordering, $bytes, 0, 0, [x1]), - // mov s(tmp0), s(0) - concat!("mov ", reg!($bytes, 16), ", ", reg!($bytes, 0)), - "0:", - // LDXR s(0), [x1] - concat!(ldxr!($ordering, $bytes), " ", reg!($bytes, 0), ", [x1]"), - // OP s(tmp1), s(0), s(tmp0) - concat!($op, " ", reg!($bytes, 17), ", ", reg!($bytes, 0), ", ", reg!($bytes, 16)), - // STXR w(tmp2), s(tmp1), [x1] - concat!(stxr!($ordering, $bytes), " w15, ", reg!($bytes, 17), ", [x1]"), - "cbnz w15, 0b", - "ret", - have_lse = sym crate::aarch64_outline_atomics::HAVE_LSE_ATOMICS, + let mut val = val; + unsafe { + if HAVE_LSE_ATOMICS.load(Ordering::Relaxed) != 0 { + core::arch::asm! { + ".arch_extension lse", + sanitize_ptr!(1), + // LSEOP s(0), s(0), [x1]; if LSE supported. + concat!(lse!($lse_op, $ordering, $bytes), " ", reg!($bytes, 0), ", ", reg!($bytes, 0),", [x1]"), + inlateout("x0") val, + inout("x1") ptr => _, + options(nostack, preserves_flags), + }; + } else { + core::arch::asm! { + sanitize_ptr!(1), + // mov s(tmp0), s(0) + concat!("mov ", reg!($bytes, 16), ", ", reg!($bytes, 0)), + "1:", + // LDXR s(0), [x1] + concat!(ldxr!($ordering, $bytes), " ", reg!($bytes, 0), ", [x1]"), + // OP s(tmp1), s(0), s(tmp0) + concat!($op, " ", reg!($bytes, 17), ", ", reg!($bytes, 0), ", ", reg!($bytes, 16)), + // STXR w(tmp2), s(tmp1), [x1] + concat!(stxr!($ordering, $bytes), " w15, ", reg!($bytes, 17), ", [x1]"), + "cbnz w15, 1b", + inlateout("x0") val, + inout("x1") ptr => _, + out("w15") _, + out("x16") _, + out("x17") _, + options(nostack, preserves_flags), + } + } } + val } } } diff --git a/library/compiler-builtins/compiler-builtins/src/float/conv.rs b/library/compiler-builtins/compiler-builtins/src/float/conv.rs index 13a0ed4fcd39f..eb6a10bfcd238 100644 --- a/library/compiler-builtins/compiler-builtins/src/float/conv.rs +++ b/library/compiler-builtins/compiler-builtins/src/float/conv.rs @@ -208,6 +208,8 @@ mod int_to_float { // Conversions from unsigned integers to floats. intrinsics! { + /* u32 -> fN */ + #[arm_aeabi_alias = __aeabi_ui2f] pub extern "C" fn __floatunsisf(i: u32) -> f32 { f32::from_bits(int_to_float::u32_to_f32_bits(i)) @@ -218,6 +220,14 @@ intrinsics! { f64::from_bits(int_to_float::u32_to_f64_bits(i)) } + #[ppc_name = __floatunsikf] + #[cfg(f128_enabled)] + pub extern "C" fn __floatunsitf(i: u32) -> f128 { + f128::from_bits(int_to_float::u32_to_f128_bits(i)) + } + + /* u64 -> fN */ + #[arm_aeabi_alias = __aeabi_ul2f] pub extern "C" fn __floatundisf(i: u64) -> f32 { f32::from_bits(int_to_float::u64_to_f32_bits(i)) @@ -228,6 +238,14 @@ intrinsics! { f64::from_bits(int_to_float::u64_to_f64_bits(i)) } + #[ppc_name = __floatundikf] + #[cfg(f128_enabled)] + pub extern "C" fn __floatunditf(i: u64) -> f128 { + f128::from_bits(int_to_float::u64_to_f128_bits(i)) + } + + /* u128 -> fN */ + #[cfg(not(all(target_os = "uefi", target_arch = "x86_64")))] pub extern "C" fn __floatuntisf(i: u128) -> f32 { f32::from_bits(int_to_float::u128_to_f32_bits(i)) @@ -248,18 +266,6 @@ intrinsics! { f64::from_bits(int_to_float::u128_to_f64_bits((u128::from(hi) << 64) | u128::from(lo))) } - #[ppc_name = __floatunsikf] - #[cfg(f128_enabled)] - pub extern "C" fn __floatunsitf(i: u32) -> f128 { - f128::from_bits(int_to_float::u32_to_f128_bits(i)) - } - - #[ppc_name = __floatundikf] - #[cfg(f128_enabled)] - pub extern "C" fn __floatunditf(i: u64) -> f128 { - f128::from_bits(int_to_float::u64_to_f128_bits(i)) - } - #[ppc_name = __floatuntikf] #[cfg(f128_enabled)] pub extern "C" fn __floatuntitf(i: u128) -> f128 { @@ -269,6 +275,8 @@ intrinsics! { // Conversions from signed integers to floats. intrinsics! { + /* i32 -> fN */ + #[arm_aeabi_alias = __aeabi_i2f] pub extern "C" fn __floatsisf(i: i32) -> f32 { int_to_float::signed(i, int_to_float::u32_to_f32_bits) @@ -279,6 +287,14 @@ intrinsics! { int_to_float::signed(i, int_to_float::u32_to_f64_bits) } + #[ppc_name = __floatsikf] + #[cfg(f128_enabled)] + pub extern "C" fn __floatsitf(i: i32) -> f128 { + int_to_float::signed(i, int_to_float::u32_to_f128_bits) + } + + /* i64 -> fN */ + #[arm_aeabi_alias = __aeabi_l2f] pub extern "C" fn __floatdisf(i: i64) -> f32 { int_to_float::signed(i, int_to_float::u64_to_f32_bits) @@ -289,6 +305,14 @@ intrinsics! { int_to_float::signed(i, int_to_float::u64_to_f64_bits) } + #[ppc_name = __floatdikf] + #[cfg(f128_enabled)] + pub extern "C" fn __floatditf(i: i64) -> f128 { + int_to_float::signed(i, int_to_float::u64_to_f128_bits) + } + + /* i128 -> fN */ + #[cfg(not(all(target_os = "uefi", target_arch = "x86_64")))] pub extern "C" fn __floattisf(i: i128) -> f32 { int_to_float::signed(i, int_to_float::u128_to_f32_bits) @@ -309,18 +333,6 @@ intrinsics! { int_to_float::signed((i128::from(hi) << 64) | i128::from(lo), int_to_float::u128_to_f64_bits) } - #[ppc_name = __floatsikf] - #[cfg(f128_enabled)] - pub extern "C" fn __floatsitf(i: i32) -> f128 { - int_to_float::signed(i, int_to_float::u32_to_f128_bits) - } - - #[ppc_name = __floatdikf] - #[cfg(f128_enabled)] - pub extern "C" fn __floatditf(i: i64) -> f128 { - int_to_float::signed(i, int_to_float::u64_to_f128_bits) - } - #[ppc_name = __floattikf] #[cfg(f128_enabled)] pub extern "C" fn __floattitf(i: i128) -> f128 { @@ -411,6 +423,8 @@ where // Conversions from floats to unsigned integers. intrinsics! { + /* f32 -> uN */ + #[arm_aeabi_alias = __aeabi_f2uiz] pub extern "C" fn __fixunssfsi(f: f32) -> u32 { float_to_unsigned_int(f) @@ -425,6 +439,8 @@ intrinsics! { float_to_unsigned_int(f) } + /* f64 -> uN */ + #[arm_aeabi_alias = __aeabi_d2uiz] pub extern "C" fn __fixunsdfsi(f: f64) -> u32 { float_to_unsigned_int(f) @@ -439,6 +455,8 @@ intrinsics! { float_to_unsigned_int(f) } + /* f128 -> uN */ + #[ppc_name = __fixunskfsi] #[cfg(f128_enabled)] pub extern "C" fn __fixunstfsi(f: f128) -> u32 { @@ -460,6 +478,8 @@ intrinsics! { // Conversions from floats to signed integers. intrinsics! { + /* f32 -> iN */ + #[arm_aeabi_alias = __aeabi_f2iz] pub extern "C" fn __fixsfsi(f: f32) -> i32 { float_to_signed_int(f) @@ -474,6 +494,8 @@ intrinsics! { float_to_signed_int(f) } + /* f64 -> iN */ + #[arm_aeabi_alias = __aeabi_d2iz] pub extern "C" fn __fixdfsi(f: f64) -> i32 { float_to_signed_int(f) @@ -488,6 +510,8 @@ intrinsics! { float_to_signed_int(f) } + /* f128 -> iN */ + #[ppc_name = __fixkfsi] #[cfg(f128_enabled)] pub extern "C" fn __fixtfsi(f: f128) -> i32 { diff --git a/library/compiler-builtins/crates/api-list-common/src/lib.rs b/library/compiler-builtins/crates/api-list-common/src/lib.rs index f6e5516a76fa6..d560e04058264 100644 --- a/library/compiler-builtins/crates/api-list-common/src/lib.rs +++ b/library/compiler-builtins/crates/api-list-common/src/lib.rs @@ -898,6 +898,7 @@ const ALL_OPERATIONS_NESTED: &[NestedOp] = &[ "erff", "exp10f", "exp2f", + "exp2m1f", "expf", "expm1f", "fabsf", @@ -947,6 +948,7 @@ const ALL_OPERATIONS_NESTED: &[NestedOp] = &[ "exp", "exp10", "exp2", + "exp2m1", "expm1", "fabs", "floor", diff --git a/library/compiler-builtins/crates/sysbench/Cargo.toml b/library/compiler-builtins/crates/sysbench/Cargo.toml new file mode 100644 index 0000000000000..acda233ba6743 --- /dev/null +++ b/library/compiler-builtins/crates/sysbench/Cargo.toml @@ -0,0 +1,7 @@ +[package] +name = "sysbench" +version = "0.1.0" +edition = "2024" +publish = false + +[dependencies] diff --git a/library/compiler-builtins/crates/sysbench/src/main.rs b/library/compiler-builtins/crates/sysbench/src/main.rs new file mode 100644 index 0000000000000..5933f2ccd740d --- /dev/null +++ b/library/compiler-builtins/crates/sysbench/src/main.rs @@ -0,0 +1,121 @@ +//! Simple check of integer and float ops, for giving us an idea of how many test cases we can +//! expect to execute in a reasonable amount of time. + +#![allow(clippy::type_complexity)] + +use std::hint::black_box as bb; +use std::io::Write; +use std::time::{Duration, Instant}; +use std::{io, thread}; + +const ESTIMATE_ITERS: u32 = 65535; +const ITER_MAX: u32 = u32::MAX; + +fn main() { + let int_mul = run_bench("integer multiplication", |i| { + let i = i128::from(i); + let i = i | (i << 32) | (i << 64) | (i << 96); + bb(i).overflowing_mul(bb(i)) + }); + let float_mul = run_bench("float multiplication", |i| { + let x = f32::from_bits(i); + bb(x) * bb(x) + }); + let float_fma = run_bench("float fma", |i| { + let x = f32::from_bits(i); + bb(x).mul_add(bb(x), bb(x)) + }); + + let oneshot = bench_oneshot(); + + let threads = thread::available_parallelism().unwrap().get(); + println!("Available parallelism: {threads}"); + + // Super simple metric to give us a rough idea of how machines compare. Int math gets a larger + // scale since u128 requires more instructions. + let int_score = 500.0 / int_mul; + let float_score = 200.0 / float_mul; + let fma_score = 200.0 / float_fma; + let oneshot_score = 50.0 / oneshot.as_secs_f64(); + let score = int_score + float_score + fma_score + oneshot_score; + let thread_scale = 1.0 + 0.9 * (threads as f64 - 1.0); + + println!("Single-core performance score: {score:.0}"); + println!("Multi-core performance score: {:.0}", score * thread_scale); +} + +/// Benchmark `f` by running it repeatedly after a warmup. Outputs are black boxed. +fn run_bench(name: &str, mut f: impl FnMut(u32) -> T) -> f64 { + print!("Starting {name} bench... "); + io::stdout().flush().unwrap(); + + // Warmup + for i in 0..1000 { + bb(f(bb(i))); + } + + // Calculate how many to run + let start = Instant::now(); + for i in 0..ESTIMATE_ITERS { + bb(f(bb(i))); + } + let elapsed = start.elapsed(); + + let ns_per_iter = elapsed.as_secs_f64() * 1e9 / ESTIMATE_ITERS as f64; + let desired_iters = 1e9 / ns_per_iter; + let step = ((ITER_MAX as u64 + 1) / desired_iters as u64) as usize; + let iter = (0..=ITER_MAX).step_by(step); + let count = iter_count(iter.clone()); + + // Run the benchmark + let start = Instant::now(); + for i in iter { + bb(f(bb(i))); + } + let elapsed = start.elapsed(); + + let ns_per_iter = elapsed.as_secs_f64() * 1e9 / count as f64; + println!("{count} iterations completed in {elapsed:.3?} ({ns_per_iter:.3} ns/iter)"); + ns_per_iter +} + +/// Qemu has a good JIT, so run something new once to try to catch it. +fn bench_oneshot() -> Duration { + fn fib_int(n: u32) -> u32 { + if n == 0 { + 0 + } else if n == 1 { + 1 + } else { + fib_int(n - 1) + fib_int(n - 2) + } + } + + fn fib_float(n: f32) -> f32 { + if n == 0.0 { + 0.0 + } else if n == 1.0 { + 1.0 + } else { + fib_float(n - 1.0) + fib_float(n - 2.0) + } + } + + print!("Starting oneshot bench... "); + io::stdout().flush().unwrap(); + let start = Instant::now(); + bb(fib_int(bb(38))); + bb(fib_float(bb(38.0))); + let elapsed = start.elapsed(); + println!("completed in {elapsed:.3?}"); + elapsed +} + +/// `iter.count()` but allow results greater than `usize::MAX` on 32-bit. +fn iter_count(iter: impl Iterator) -> u64 { + let mut ret = 0u64; + for _ in iter { + ret = ret.strict_add(1); + } + ret +} diff --git a/library/compiler-builtins/crates/util/src/main.rs b/library/compiler-builtins/crates/util/src/main.rs index aa79354a57159..4cc6065bab63f 100644 --- a/library/compiler-builtins/crates/util/src/main.rs +++ b/library/compiler-builtins/crates/util/src/main.rs @@ -102,7 +102,9 @@ fn do_eval(basis: &str, op: &str, inputs: &[&str]) { extra: (basis, op, inputs), fn_extra: match MACRO_FN_NAME { // Not provided by musl - fmaximum + exp2m1 + | exp2m1f + | fmaximum | fmaximum_num | fmaximum_numf | fmaximumf diff --git a/library/compiler-builtins/etc/function-definitions.json b/library/compiler-builtins/etc/function-definitions.json index 3e2b14d31c53b..fecb646a0f544 100644 --- a/library/compiler-builtins/etc/function-definitions.json +++ b/library/compiler-builtins/etc/function-definitions.json @@ -233,6 +233,18 @@ ], "type": "f32" }, + "exp2m1": { + "sources": [ + "libm/src/math/exp2m1.rs" + ], + "type": "f64" + }, + "exp2m1f": { + "sources": [ + "libm/src/math/exp2m1f.rs" + ], + "type": "f32" + }, "expf": { "sources": [ "libm/src/math/expf.rs" diff --git a/library/compiler-builtins/etc/function-list.txt b/library/compiler-builtins/etc/function-list.txt index 14f75a2c5783b..b65540fb18e4d 100644 --- a/library/compiler-builtins/etc/function-list.txt +++ b/library/compiler-builtins/etc/function-list.txt @@ -36,6 +36,8 @@ exp10 exp10f exp2 exp2f +exp2m1 +exp2m1f expf expm1 expm1f diff --git a/library/compiler-builtins/libm-test/benches/icount.rs b/library/compiler-builtins/libm-test/benches/icount.rs index 8e067f2da5e6d..e9eb3805d1d78 100644 --- a/library/compiler-builtins/libm-test/benches/icount.rs +++ b/library/compiler-builtins/libm-test/benches/icount.rs @@ -108,6 +108,8 @@ library_benchmark_group!( icount_bench_exp10f, icount_bench_exp2, icount_bench_exp2f, + icount_bench_exp2m1, + icount_bench_exp2m1f, icount_bench_expf, icount_bench_expm1, icount_bench_expm1f, diff --git a/library/compiler-builtins/libm-test/benches/random.rs b/library/compiler-builtins/libm-test/benches/random.rs index ae4c2f51bdec5..d6ee71f9b3826 100644 --- a/library/compiler-builtins/libm-test/benches/random.rs +++ b/library/compiler-builtins/libm-test/benches/random.rs @@ -126,7 +126,9 @@ libm_macros::for_each_function! { exp10 | exp10f | exp2 | exp2f => (true, Some(musl_math_sys::MACRO_FN_NAME)), // Musl does not provide `f16` and `f128` functions, as well as a handful of others - fmaximum + exp2m1 + | exp2m1f + | fmaximum | fmaximum_num | fmaximum_numf | fmaximumf diff --git a/library/compiler-builtins/libm-test/src/builtins_wrapper.rs b/library/compiler-builtins/libm-test/src/builtins_wrapper.rs index ce517148b9ca3..80716619f9c5e 100644 --- a/library/compiler-builtins/libm-test/src/builtins_wrapper.rs +++ b/library/compiler-builtins/libm-test/src/builtins_wrapper.rs @@ -102,80 +102,128 @@ cb_op!(@binop f16, add, __addhf3, addf16); cb_op!(@binop f32, add, __addsf3, addf32); cb_op!(@binop f64, add, __adddf3, addf64); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(@binop f128, add, __addtf3, addf128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(@binop f128, add, __addkf3, addf128); #[cfg(f16_enabled)] cb_op!(@binop f16, sub, __subhf3, subf16); cb_op!(@binop f32, sub, __subsf3, subf32); cb_op!(@binop f64, sub, __subdf3, subf64); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(@binop f128, sub, __subtf3, subf128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(@binop f128, sub, __subkf3, subf128); #[cfg(f16_enabled)] cb_op!(@binop f16, mul, __mulhf3, mulf16); cb_op!(@binop f32, mul, __mulsf3, mulf32); cb_op!(@binop f64, mul, __muldf3, mulf64); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(@binop f128, mul, __multf3, mulf128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(@binop f128, mul, __mulkf3, mulf128); cb_op!(@binop f32, div, __divsf3, divf32); cb_op!(@binop f64, div, __divdf3, divf64); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(@binop f128, div, __divtf3, divf128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(@binop f128, div, __divkf3, divf128); cb_op!(pow, __powisf2, powif32, (a: f32, b: i32) -> f32); cb_op!(pow, __powidf2, powif64, (a: f64, b: i32) -> f64); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(pow, __powitf2, powif128, (a: f128, b: i32) -> f128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(pow, __powikf2, powif128, (a: f128, b: i32) -> f128); #[cfg(f16_enabled)] cb_op!(@cmp_eq f16, cmp, __eqhf2, eqf16); cb_op!(@cmp_eq f32, cmp, __eqsf2, eqf32); cb_op!(@cmp_eq f64, cmp, __eqdf2, eqf64); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(@cmp_eq f128, cmp, __eqtf2, eqf128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(@cmp_eq f128, cmp, __eqkf2, eqf128); #[cfg(f16_enabled)] cb_op!(@cmp_gt f16, cmp, __gthf2, gtf16); cb_op!(@cmp_gt f32, cmp, __gtsf2, gtf32); cb_op!(@cmp_gt f64, cmp, __gtdf2, gtf64); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(@cmp_gt f128, cmp, __gttf2, gtf128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(@cmp_gt f128, cmp, __gtkf2, gtf128); #[cfg(f16_enabled)] cb_op!(@cmp_ge f16, cmp, __gehf2, gef16); cb_op!(@cmp_ge f32, cmp, __gesf2, gef32); cb_op!(@cmp_ge f64, cmp, __gedf2, gef64); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(@cmp_ge f128, cmp, __getf2, gef128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(@cmp_ge f128, cmp, __gekf2, gef128); #[cfg(f16_enabled)] cb_op!(@cmp_lt f16, cmp, __lthf2, ltf16); cb_op!(@cmp_lt f32, cmp, __ltsf2, ltf32); cb_op!(@cmp_lt f64, cmp, __ltdf2, ltf64); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(@cmp_lt f128, cmp, __lttf2, ltf128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(@cmp_lt f128, cmp, __ltkf2, ltf128); #[cfg(f16_enabled)] cb_op!(@cmp_le f16, cmp, __lehf2, lef16); cb_op!(@cmp_le f32, cmp, __lesf2, lef32); cb_op!(@cmp_le f64, cmp, __ledf2, lef64); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(@cmp_le f128, cmp, __letf2, lef128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(@cmp_le f128, cmp, __lekf2, lef128); #[cfg(f16_enabled)] cb_op!(@cmp_ne f16, cmp, __nehf2, nef16); cb_op!(@cmp_ne f32, cmp, __nesf2, nef32); cb_op!(@cmp_ne f64, cmp, __nedf2, nef64); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(@cmp_ne f128, cmp, __netf2, nef128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(@cmp_ne f128, cmp, __nekf2, nef128); #[cfg(f16_enabled)] cb_op!(@cmp_unord f16, cmp, __unordhf2, unordf16); cb_op!(@cmp_unord f32, cmp, __unordsf2, unordf32); cb_op!(@cmp_unord f64, cmp, __unorddf2, unordf64); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(@cmp_unord f128, cmp, __unordtf2, unordf128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(@cmp_unord f128, cmp, __unordkf2, unordf128); #[cfg(f16_enabled)] cb_op!(extend, __extendhfsf2, extend_f16_f32, (a: f16) -> f32); @@ -183,12 +231,25 @@ cb_op!(extend, __extendhfsf2, extend_f16_f32, (a: f16) -> f32); cb_op!(extend, __extendhfdf2, extend_f16_f64, (a: f16) -> f64); #[cfg(f16_enabled)] #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(extend, __extendhftf2, extend_f16_f128, (a: f16) -> f128); +#[cfg(f16_enabled)] +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(extend, __extendhfkf2, extend_f16_f128, (a: f16) -> f128); cb_op!(extend, __extendsfdf2, extend_f32_f64, (a: f32) -> f64); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(extend, __extendsftf2, extend_f32_f128, (a: f32) -> f128); #[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(extend, __extendsfkf2, extend_f32_f128, (a: f32) -> f128); +#[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(extend, __extenddftf2, extend_f64_f128, (a: f64) -> f128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(extend, __extenddfkf2, extend_f64_f128, (a: f64) -> f128); // Note that these are renamed from trunc to narrow to avoid collision with libm `trunc`. #[cfg(f16_enabled)] @@ -198,11 +259,23 @@ cb_op!(trunc, __truncdfhf2, narrow_f64_f16, (a: f64) -> f16); cb_op!(trunc, __truncdfsf2, narrow_f64_f32, (a: f64) -> f32); #[cfg(f16_enabled)] #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(trunc, __trunctfhf2, narrow_f128_f16, (a: f128) -> f16); #[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(trunc, __trunckfhf2, narrow_f128_f16, (a: f128) -> f16); +#[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(trunc, __trunctfsf2, narrow_f128_f32, (a: f128) -> f32); #[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(trunc, __trunckfsf2, narrow_f128_f32, (a: f128) -> f32); +#[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(trunc, __trunctfdf2, narrow_f128_f64, (a: f128) -> f64); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(trunc, __trunckfdf2, narrow_f128_f64, (a: f128) -> f64); cb_op!(conv, __fixsfsi, ftoi_f32_i32, (a: f32) -> i32); cb_op!(conv, __fixsfdi, ftoi_f32_i64, (a: f32) -> i64); @@ -211,11 +284,23 @@ cb_op!(conv, __fixdfsi, ftoi_f64_i32, (a: f64) -> i32); cb_op!(conv, __fixdfdi, ftoi_f64_i64, (a: f64) -> i64); cb_op!(conv, __fixdfti, ftoi_f64_i128, (a: f64) -> i128); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(conv, __fixtfsi, ftoi_f128_i32, (a: f128) -> i32); #[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(conv, __fixkfsi, ftoi_f128_i32, (a: f128) -> i32); +#[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(conv, __fixtfdi, ftoi_f128_i64, (a: f128) -> i64); #[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(conv, __fixkfdi, ftoi_f128_i64, (a: f128) -> i64); +#[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(conv, __fixtfti, ftoi_f128_i128, (a: f128) -> i128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(conv, __fixkfti, ftoi_f128_i128, (a: f128) -> i128); cb_op!(conv, __fixunssfsi, ftoi_f32_u32, (a: f32) -> u32); cb_op!(conv, __fixunssfdi, ftoi_f32_u64, (a: f32) -> u64); cb_op!(conv, __fixunssfti, ftoi_f32_u128, (a: f32) -> u128); @@ -223,11 +308,23 @@ cb_op!(conv, __fixunsdfsi, ftoi_f64_u32, (a: f64) -> u32); cb_op!(conv, __fixunsdfdi, ftoi_f64_u64, (a: f64) -> u64); cb_op!(conv, __fixunsdfti, ftoi_f64_u128, (a: f64) -> u128); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(conv, __fixunstfsi, ftoi_f128_u32, (a: f128) -> u32); #[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(conv, __fixunskfsi, ftoi_f128_u32, (a: f128) -> u32); +#[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(conv, __fixunstfdi, ftoi_f128_u64, (a: f128) -> u64); #[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(conv, __fixunskfdi, ftoi_f128_u64, (a: f128) -> u64); +#[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(conv, __fixunstfti, ftoi_f128_u128, (a: f128) -> u128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(conv, __fixunskfti, ftoi_f128_u128, (a: f128) -> u128); cb_op!(conv, __floatsisf, itof_i32_f32, (a: i32) -> f32); cb_op!(conv, __floatdisf, itof_i64_f32, (a: i64) -> f32); @@ -236,11 +333,23 @@ cb_op!(conv, __floatsidf, itof_i32_f64, (a: i32) -> f64); cb_op!(conv, __floatdidf, itof_i64_f64, (a: i64) -> f64); cb_op!(conv, __floattidf, itof_i128_f64, (a: i128) -> f64); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(conv, __floatsitf, itof_i32_f128, (a: i32) -> f128); #[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(conv, __floatsikf, itof_i32_f128, (a: i32) -> f128); +#[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(conv, __floatditf, itof_i64_f128, (a: i64) -> f128); #[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(conv, __floatdikf, itof_i64_f128, (a: i64) -> f128); +#[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(conv, __floattitf, itof_i128_f128, (a: i128) -> f128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(conv, __floattikf, itof_i128_f128, (a: i128) -> f128); cb_op!(conv, __floatunsisf, itof_u32_f32, (a: u32) -> f32); cb_op!(conv, __floatundisf, itof_u64_f32, (a: u64) -> f32); cb_op!(conv, __floatuntisf, itof_u128_f32, (a: u128) -> f32); @@ -248,11 +357,23 @@ cb_op!(conv, __floatunsidf, itof_u32_f64, (a: u32) -> f64); cb_op!(conv, __floatundidf, itof_u64_f64, (a: u64) -> f64); cb_op!(conv, __floatuntidf, itof_u128_f64, (a: u128) -> f64); #[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(conv, __floatunsitf, itof_u32_f128, (a: u32) -> f128); #[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(conv, __floatunsikf, itof_u32_f128, (a: u32) -> f128); +#[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(conv, __floatunditf, itof_u64_f128, (a: u64) -> f128); #[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(conv, __floatundikf, itof_u64_f128, (a: u64) -> f128); +#[cfg(f128_enabled)] +#[cfg(not(any(target_arch = "powerpc", target_arch = "powerpc64")))] cb_op!(conv, __floatuntitf, itof_u128_f128, (a: u128) -> f128); +#[cfg(f128_enabled)] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] +cb_op!(conv, __floatuntikf, itof_u128_f128, (a: u128) -> f128); /* int ops */ diff --git a/library/compiler-builtins/libm-test/src/domain.rs b/library/compiler-builtins/libm-test/src/domain.rs index 0ae105f4a02d2..f37e50cc08da7 100644 --- a/library/compiler-builtins/libm-test/src/domain.rs +++ b/library/compiler-builtins/libm-test/src/domain.rs @@ -283,6 +283,7 @@ pub fn get_domain( BaseName::Expm1 => &EitherPrim::UNBOUNDED1[..], BaseName::Exp10 => &EitherPrim::UNBOUNDED1[..], BaseName::Exp2 => &EitherPrim::UNBOUNDED1[..], + BaseName::Exp2m1 => &EitherPrim::UNBOUNDED1[..], BaseName::Frexp => &EitherPrim::UNBOUNDED1[..], BaseName::Fabs => &EitherPrim::UNBOUNDED1[..], BaseName::Fdim => &EitherPrim::UNBOUNDED2[..], diff --git a/library/compiler-builtins/libm-test/src/f8_impl.rs b/library/compiler-builtins/libm-test/src/f8_impl.rs index 7a24fcbfa3c86..ad968d7ddd4d7 100644 --- a/library/compiler-builtins/libm-test/src/f8_impl.rs +++ b/library/compiler-builtins/libm-test/src/f8_impl.rs @@ -97,6 +97,14 @@ impl Float for f8 { unimplemented!() } + fn floor(self) -> Self { + unimplemented!() + } + + fn roundeven(self) -> Self { + unimplemented!() + } + fn normalize(_significand: Self::Int) -> (i32, Self::Int) { unimplemented!() } diff --git a/library/compiler-builtins/libm-test/src/generate/case_list.rs b/library/compiler-builtins/libm-test/src/generate/case_list.rs index 2275787307d95..353b49b87f4ae 100644 --- a/library/compiler-builtins/libm-test/src/generate/case_list.rs +++ b/library/compiler-builtins/libm-test/src/generate/case_list.rs @@ -12,7 +12,7 @@ use libm::hf16; use libm::hf128; use libm::{hf32, hf64}; -use crate::{CheckBasis, CheckCtx, GeneratorKind, MathOp, op}; +use crate::{CheckBasis, CheckCtx, Float, GeneratorKind, MathOp, op}; pub struct TestCase { pub input: Op::RustArgs, @@ -852,6 +852,122 @@ fn exp2f_cases() -> Vec> { cases![] } +fn exp2m1f_cases() -> Vec> { + let mut cases = cases![]; + + // Boundary values from CORE-MATH's binary32/exp2m1/exp2m1f.c. + // Test each value and its neighbors with both signs against MPFR. + for bits in [ + 0x0000_0001, + 0x0080_0000, + 0x2538_aa3b, + 0x331f_dd82, + 0x37d3_2ef6, + 0x3a35_8876, + 0x3bac_1405, + 0x3caa_2fee, + 0x3d67_a4cc, + 0x3df9_5f1f, + 0x41c8_0000, + 0x4300_0000, + ] { + for adjacent in [bits - 1, bits, bits + 1] { + for sign in [0, f32::SIGN_MASK] { + cases.push(TestCase { + input: (f32::from_bits(adjacent | sign),), + output: None, + }); + } + } + } + + // Special rounding cases from the same CORE-MATH file. + // Test each value and its neighbors against MPFR. + for bits in [0xb3d8_5005, 0x3338_428d, 0x388b_ca4f] { + for adjacent in [bits - 1, bits, bits + 1] { + cases.push(TestCase { + input: (f32::from_bits(adjacent),), + output: None, + }); + } + } + + cases +} + +fn exp2m1_cases() -> Vec> { + let mut cases = cases![]; + + // Check their neighbors with MPFR. + for i in 1..=53 { + let x = i as f64; + + for adjacent in [x.to_bits() - 1, x.to_bits() + 1] { + for sign in [0, f64::SIGN_MASK] { + cases.push(TestCase { + input: (f64::from_bits(adjacent | sign),), + output: None, + }); + } + } + } + + // Subnormal, polynomial, saturation, and overflow boundaries, with both signs. + for x in [ + f64::from_bits(1), + f64::MIN_POSITIVE, + hf64!("0x1.71547652b82fdp-1022"), + hf64!("0x1p-104"), + hf64!("0x1.0527dbd87e24dp-51"), + 0.125, + 54.0, + 1024.0, + ] { + let bits = x.to_bits(); + for adjacent in [bits - 1, bits, bits + 1] { + for sign in [0, f64::SIGN_MASK] { + cases.push(TestCase { + input: (f64::from_bits(adjacent | sign),), + output: None, + }); + } + } + } + + // Difficult rounding cases from the CORE-MATH exception tables and their neighbors. + for x in [ + hf64!("-0x1.a16826a8e825dp-56"), + hf64!("-0x1.f1bc3ef3e6f36p-65"), + hf64!("-0x1.1e3a6eaa49c6ep-84"), + hf64!("-0x1.1d65d5fc31246p-98"), + hf64!("0x1.8a5dd21ef35afp-104"), + hf64!("0x1.086cbb0e900cfp-96"), + hf64!("0x1.453f5d718374ap-88"), + hf64!("0x1.02f371449097fp-51"), + hf64!("-0x1.f6ec73d3948c3p-4"), + hf64!("-0x1.3918e8608bd5bp-8"), + hf64!("-0x1.2c506b0368099p-51"), + hf64!("0x1.391609b20beaap-51"), + hf64!("0x1.4753a08baf7fbp-22"), + hf64!("0x1.d4a8ebce833a3p-4"), + hf64!("-0x1.da22611253866p+0"), + hf64!("-0x1.8827da1dbf35ep-2"), + hf64!("0x1.12eecf76d63cdp+1"), + hf64!("0x1.ffef3f31766b7p+5"), + hf64!("0x1.6f33cdaf56d6p+6"), + ] { + let bits = x.to_bits(); + for adjacent in [bits - 1, bits, bits + 1] { + cases.push(TestCase { + input: (f64::from_bits(adjacent),), + output: None, + }); + } + } + + cases +} + fn expf_cases() -> Vec> { cases![ ((hf32!("-0x1.2d245ap-8")), hf32!("0x1.fda718p-1")), diff --git a/library/compiler-builtins/libm-test/src/mpfloat.rs b/library/compiler-builtins/libm-test/src/mpfloat.rs index 69c6590d3763b..5517e70eb010b 100644 --- a/library/compiler-builtins/libm-test/src/mpfloat.rs +++ b/library/compiler-builtins/libm-test/src/mpfloat.rs @@ -355,6 +355,7 @@ libm_macros::for_each_function! { fn_extra: match MACRO_FN_NAME { // Remap function names that are different between mpfr and libm expm1 | expm1f => exp_m1, + exp2m1 | exp2m1f => exp2_m1, fabs | fabsf => abs, fdim | fdimf | fdimf16 | fdimf128 => positive_diff, fmaf16 | fma | fmaf | fmaf128 => mul_add, diff --git a/library/compiler-builtins/libm-test/src/num.rs b/library/compiler-builtins/libm-test/src/num.rs index efa87d6351597..4b7e17d48c9e2 100644 --- a/library/compiler-builtins/libm-test/src/num.rs +++ b/library/compiler-builtins/libm-test/src/num.rs @@ -11,35 +11,11 @@ use crate::{Int, MinInt}; /// Extension to `libm`'s `Float` trait with methods that are useful for tests but not /// needed in `libm` itself. pub trait FloatExt: Float { - /// The minimum subnormal number. - const TINY_BITS: Self::Int = Self::Int::ONE; - /// Retrieve additional constants for this float type. fn consts() -> Consts { Consts::new() } - /// Increment by one ULP, saturating at infinity. - fn next_up(self) -> Self { - let bits = self.to_bits(); - if self.is_nan() || bits == Self::INFINITY.to_bits() { - return self; - } - - let abs = self.abs().to_bits(); - let next_bits = if abs == Self::Int::ZERO { - // Next up from 0 is the smallest subnormal - Self::TINY_BITS - } else if bits == abs { - // Positive: counting up is more positive - bits + Self::Int::ONE - } else { - // Negative: counting down is more positive - bits - Self::Int::ONE - }; - Self::from_bits(next_bits) - } - /// A faster way to effectively call `next_up` `n` times. fn n_up(self, n: Self::Int) -> Self { let bits = self.to_bits(); @@ -70,27 +46,6 @@ pub trait FloatExt: Float { Self::from_bits(next_bits) } - /// Decrement by one ULP, saturating at negative infinity. - fn next_down(self) -> Self { - let bits = self.to_bits(); - if self.is_nan() || bits == Self::NEG_INFINITY.to_bits() { - return self; - } - - let abs = self.abs().to_bits(); - let next_bits = if abs == Self::Int::ZERO { - // Next up from 0 is the smallest negative subnormal - Self::TINY_BITS | Self::SIGN_MASK - } else if bits == abs { - // Positive: counting down is more negative - bits - Self::Int::ONE - } else { - // Negative: counting up is more negative - bits + Self::Int::ONE - }; - Self::from_bits(next_bits) - } - /// A faster way to effectively call `next_down` `n` times. fn n_down(self, n: Self::Int) -> Self { let bits = self.to_bits(); diff --git a/library/compiler-builtins/libm-test/src/precision.rs b/library/compiler-builtins/libm-test/src/precision.rs index 967274caa9644..1acbfe8dbedb8 100644 --- a/library/compiler-builtins/libm-test/src/precision.rs +++ b/library/compiler-builtins/libm-test/src/precision.rs @@ -76,6 +76,7 @@ pub fn default_ulp(ctx: &CheckCtx) -> Option { // Operations that aren't required to be exact, but our implementations are. Bn::Cbrt => 0, Bn::Hypot if ctx.fn_ident == Id::Hypot => 0, + Bn::Exp2m1 => 0, // Bessel functions have large inaccuracies. Bn::J0 | Bn::J1 | Bn::Y0 | Bn::Y1 | Bn::Jn | Bn::Yn => 8_000_000, @@ -148,6 +149,7 @@ pub fn default_ulp(ctx: &CheckCtx) -> Option { Id::Log1p => ulp = 2, Id::Log1pf => ulp = 2, Id::Tan => ulp = 2, + Id::Exp2m1 => ulp = 512, _ => (), } diff --git a/library/compiler-builtins/libm-test/tests/compare_built_musl.rs b/library/compiler-builtins/libm-test/tests/compare_built_musl.rs index 5e9ead0473c2e..ec09e16cf24cc 100644 --- a/library/compiler-builtins/libm-test/tests/compare_built_musl.rs +++ b/library/compiler-builtins/libm-test/tests/compare_built_musl.rs @@ -92,6 +92,8 @@ libm_macros::for_each_function! { // Not provided by musl // verify-sorted-start + exp2m1, + exp2m1f, fmaximum, fmaximum_num, fmaximum_numf, diff --git a/library/compiler-builtins/libm-test/tests/u256.rs b/library/compiler-builtins/libm-test/tests/u256.rs index a17e16f8f429b..0163bd3bec7d0 100644 --- a/library/compiler-builtins/libm-test/tests/u256.rs +++ b/library/compiler-builtins/libm-test/tests/u256.rs @@ -322,7 +322,6 @@ fn mp_i256_sub() { let y = random_i256(&mut rng); assign_bigint_i256(&mut bx, x); assign_bigint_i256(&mut by, y); - dbg!(&bx, &by); // Emulate wrapping semantics with panicking ops let actual = if y > i256::ZERO && x < i256::MIN + y { diff --git a/library/compiler-builtins/libm/src/libm_helper.rs b/library/compiler-builtins/libm/src/libm_helper.rs index f2c4daed561ec..16b5c7e6f6704 100644 --- a/library/compiler-builtins/libm/src/libm_helper.rs +++ b/library/compiler-builtins/libm/src/libm_helper.rs @@ -66,6 +66,7 @@ libm_helper! { (fn exp(x: f32) -> (f32); => expf); (fn exp10(x: f32) -> (f32); => exp10f); (fn exp2(x: f32) -> (f32); => exp2f); + (fn exp2m1(x: f32) -> (f32); => exp2m1f); (fn expm1(x: f32) -> (f32); => expm1f); (fn fabs(x: f32) -> (f32); => fabsf); (fn fdim(x: f32, y: f32) -> (f32); => fdimf); @@ -132,6 +133,7 @@ libm_helper! { (fn exp(x: f64) -> (f64); => exp); (fn exp10(x: f64) -> (f64); => exp10); (fn exp2(x: f64) -> (f64); => exp2); + (fn exp2m1(x: f64) -> (f64); => exp2m1); (fn expm1(x: f64) -> (f64); => expm1); (fn fabs(x: f64) -> (f64); => fabs); (fn fdim(x: f64, y: f64) -> (f64); => fdim); diff --git a/library/compiler-builtins/libm/src/math/arch/x86/fma.rs b/library/compiler-builtins/libm/src/math/arch/x86/fma.rs index 43ac187792d84..64957612d21d7 100644 --- a/library/compiler-builtins/libm/src/math/arch/x86/fma.rs +++ b/library/compiler-builtins/libm/src/math/arch/x86/fma.rs @@ -8,6 +8,17 @@ use crate::support::Round; use crate::support::feature_detect::select_once; pub fn fma(x: f64, y: f64, z: f64) -> f64 { + // Skips runtime detection when a feature is enabled at compile time, so the call can be + // inlined. + if cfg!(target_feature = "fma") { + // SAFETY: `fma` is enabled at compile time. + return unsafe { fma_with_fma(x, y, z) }; + } + if cfg!(target_feature = "fma4") { + // SAFETY: `fma4` is enabled at compile time. + return unsafe { fma_with_fma4(x, y, z) }; + } + select_once! { sig: fn(x: f64, y: f64, z: f64) -> f64, init: || { @@ -26,6 +37,17 @@ pub fn fma(x: f64, y: f64, z: f64) -> f64 { } pub fn fmaf(x: f32, y: f32, z: f32) -> f32 { + // Skips runtime detection when a feature is enabled at compile time, so the call can be + // inlined. + if cfg!(target_feature = "fma") { + // SAFETY: `fma` is enabled at compile time. + return unsafe { fmaf_with_fma(x, y, z) }; + } + if cfg!(target_feature = "fma4") { + // SAFETY: `fma4` is enabled at compile time. + return unsafe { fmaf_with_fma4(x, y, z) }; + } + select_once! { sig: fn(x: f32, y: f32, z: f32) -> f32, init: || { diff --git a/library/compiler-builtins/libm/src/math/exp2m1.rs b/library/compiler-builtins/libm/src/math/exp2m1.rs new file mode 100644 index 0000000000000..6cf60a2343c60 --- /dev/null +++ b/library/compiler-builtins/libm/src/math/exp2m1.rs @@ -0,0 +1,1027 @@ +/* SPDX-License-Identifier: MIT */ +/* origin: core-math/src/binary64/exp2m1/exp2m1.c + * Copyright (c) 2022-2025 Paul Zimmermann, Tom Hubrecht and Claude-Pierre Jeannerod + * Ported to Rust in 2026, malezjaa + * Approximate CORE-MATH commit: ac1993d82a2f83eadc045cb587fa8428c9ebaa16 + */ + +use super::ldexp; +use super::support::{CastFrom, CastInto, Float, Int, cold_path}; + +const LN2H: f64 = hf64!("0x1.62e42fefa39efp-1"); +const LN2L: f64 = hf64!("0x1.abc9e3b39803fp-56"); + +/// Base-2 exponential minus one (f64) +/// +/// Calculate `2^x - 1`, that is, 2 raised to the power `x` minus `1`. +#[cfg_attr(assert_no_panic, no_panic::no_panic)] +pub fn exp2m1(x: f64) -> f64 { + cr_exp2m1(x) +} + +fn cr_exp2m1(x: f64) -> f64 { + let ux = x.to_bits(); + let ax = ux & 0x7fffffffffffffff; + + if ux >= 0xc04b000000000000 { + cold_path(); + + // x = -NaN or x <= -54 + if (ux >> 52) == 0xfff { + // -NaN or -Inf + return if ux > 0xfff0000000000000 { x + x } else { -1.0 }; + } + + // for x <= -54, exp2m1(x) rounds to -1 to nearest + return -1.0 + hf64!("0x1p-54"); + } else if ax >= 0x4090000000000000 { + cold_path(); + // x = +NaN or x >= 1024 + if (ux >> 52) == 0x7ff { + // +NaN + return x + x; + } + + // for x >= 1024, exp2m1(x) rounds to +Inf to nearest, + // but for RNDZ/RNDD, we should have no overflow for x=1024 + + return hf64!("0x1.fffffffffffffp+1023") + x * hf64!("0x1.fffffffffffffp+960"); + + // |x| <= 0x1.0527dbd87e24dp-51 + } else if ax <= 0x3cc0527dbd87e24d + /* then the second term of the Taylor expansion of 2^x-1 at x=0 is + smaller in absolute value than 1/2 ulp(first term): + log(2)*x + log(2)^2*x^2/2 + ... */ + { + // we use special code when log(2)*|x| is very small, in which case + // the double-double approximation hi+lo has its lower part lo + // "truncated" + // |x| <= 2^-104 + if ax <= 0x3970000000000000 { + // special case for 0 + if x == 0.0 { + return x; + } + + // scale x by 2^106 + let x = x * hf64!("0x1p106"); + let (hi, lo) = a_mul(LN2H, x); + let lo = LN2L.fma(x, lo); + let h2 = hi + lo; // round to 53-bit precision + // scale back, hoping to avoid double rounding + let h2 = h2 * hf64!("0x1p-106"); + + // now subtract back h2 * 2^106 from hi to get the correction term + // add lo + let hi = (-h2).fma(hf64!("0x1p106"), hi) + lo; + + // add h2 + hi * 2^-106. Warning: when hi=0, 2^-106*h2 might be exact, + // thus no underflow will be raised. We have underflow for + // 0 < x <= 0x1.71547652b82fep-1022 for RNDZ, and for + // 0 < x <= 0x1.71547652b82fdp-1022 for RNDN/RNDU. + let res = hi.fma(hf64!("0x1p-106"), h2); + + if ax <= 0x171547652b82fd || res.abs() < hf64!("0x1p-1022") { + force_eval!(res * res); // underflow + } + + return res; + + // 2^-104 < |x| <= 0x1.0527dbd87e24dp-51 + } else { + // The following exceptional cases have at least 51 identical bits after + // the round bit, thus are hard to correctly round with double-double + // arithmetic. They should be sorted by increasing values of the first + // entry (x). + #[rustfmt::skip] + static EXCEPTIONS_TABLE: [(f64, f64, f64); 56] = [ + (hf64!("-0x1.a16826a8e825dp-56"), hf64!("-0x1.21530a306cc85p-56"), hf64!("-0x1.38ac4a67cep-161")), + (hf64!("-0x1.8c525b64ed08ep-59"), hf64!("-0x1.12b592f889516p-59"), hf64!("-0x1.7b71d1eep-169")), + (hf64!("-0x1.0ede4c1293a9bp-59"), hf64!("-0x1.7780d5e5cf5c5p-60"), hf64!("-0x1.6b820f41efd9p-166")), + (hf64!("-0x1.bacdbd3005cd7p-60"), hf64!("-0x1.32ed98e196cf5p-60"), hf64!("-0x1.9b32a24b8p-166")), + (hf64!("-0x1.0481d96a2dfcap-61"), hf64!("-0x1.6923c31228cd3p-62"), hf64!("-0x1.59432bd8e301p-169")), + (hf64!("-0x1.103fc46963aafp-62"), hf64!("-0x1.796ad95f38488p-63"), hf64!("-0x1.efd4112076edap-170")), + (hf64!("-0x1.f1bc3ef3e6f36p-65"), hf64!("-0x1.5900fbf46981dp-65"), hf64!("-0x1.fffffffffffffp-119")), + (hf64!("-0x1.d8bedc057858cp-65"), hf64!("-0x1.47aea7608c02bp-65"), hf64!("-0x1.ef8a5d5p-172")), + (hf64!("-0x1.986d43391ffdp-65"), hf64!("-0x1.1b19925f9fc06p-65"), hf64!("-0x1.1aada6205370bp-170")), + (hf64!("-0x1.6a480c34c7c5bp-70"), hf64!("-0x1.f63a8ce2364f3p-71"), hf64!("-0x1.11e154c338558p-177")), + (hf64!("-0x1.03eda6663a4b2p-70"), hf64!("-0x1.6856506d8234ap-71"), hf64!("-0x1.f48368854ffd7p-178")), + (hf64!("-0x1.ec44ae4bc644p-74"), hf64!("-0x1.5536e12eb7335p-74"), hf64!("-0x1.fb8acp-181")), + (hf64!("-0x1.af82b29eef2ecp-74"), hf64!("-0x1.2b19ae19e4f3ep-74"), hf64!("-0x1.730d414c3032cp-180")), + (hf64!("-0x1.0e8e362e16fe6p-74"), hf64!("-0x1.7711d03d5c7bdp-75"), hf64!("-0x1.63020ffb5bda3p-181")), + (hf64!("-0x1.5754d81696c42p-76"), hf64!("-0x1.dbf55aa9a7eb9p-77"), hf64!("-0x1.c81798a2dea9ep-184")), + (hf64!("-0x1.e2b5692bd0c53p-80"), hf64!("-0x1.4e968fb1b5f41p-80"), hf64!("0x1.fffffffffffffp-134")), + (hf64!("-0x1.0042796d534fdp-81"), hf64!("-0x1.6340571968b67p-82"), hf64!("-0x1.91fa30638e3b8p-188")), + (hf64!("-0x1.1e3a6eaa49c6ep-84"), hf64!("-0x1.8ccbeeaab37e4p-85"), hf64!("0x1p-138")), + (hf64!("-0x1.19f9b6ddcc3e9p-84"), hf64!("-0x1.86e6a6125ee5fp-85"), hf64!("-0x1.d80acb1fd9a48p-191")), + (hf64!("-0x1.9b219a1974289p-86"), hf64!("-0x1.1cf977003bdd6p-86"), hf64!("-0x1.2a8a589db0f16p-194")), + (hf64!("-0x1.b102d7663223fp-87"), hf64!("-0x1.2c23f2bc02277p-87"), hf64!("-0x1.1304b8a0f41cdp-194")), + (hf64!("-0x1.3db1f733be456p-87"), hf64!("-0x1.b86b45d2c7c74p-88"), hf64!("-0x1.65075b3baa854p-194")), + (hf64!("-0x1.712c6b2a267f5p-90"), hf64!("-0x1.ffc87ce076dfap-91"), hf64!("-0x1.922d1954bd733p-196")), + (hf64!("-0x1.8bf6e9484dd46p-91"), hf64!("-0x1.127630515eb89p-91"), hf64!("-0x1.aa2e8ae4b621cp-198")), + (hf64!("-0x1.56fe8536a47e9p-91"), hf64!("-0x1.db7daf1e016dfp-92"), hf64!("-0x1.5a5357cd9fecap-199")), + (hf64!("-0x1.4abb72eb058bep-91"), hf64!("-0x1.ca7e01c959788p-92"), hf64!("-0x1.9c974667cda43p-198")), + (hf64!("-0x1.cf23f6232620ep-94"), hf64!("-0x1.4106468e7b671p-94"), hf64!("-0x1.a1de1e34f2642p-200")), + (hf64!("-0x1.0c197520b26f4p-94"), hf64!("-0x1.73aa2cd72b7ccp-95"), hf64!("-0x1.1p-148")), + (hf64!("-0x1.83266a65e67b3p-95"), hf64!("-0x1.0c5a1aeb10a7cp-95"), hf64!("-0x1.5d790953d37ccp-202")), + (hf64!("-0x1.e48acc7d41976p-97"), hf64!("-0x1.4fdbea8f31897p-97"), hf64!("-0x1.889c4c9736da4p-202")), + (hf64!("-0x1.241e7c1327b2ap-97"), hf64!("-0x1.94f6896c09e68p-98"), hf64!("-0x1.ad2fd62d965b7p-203")), + (hf64!("-0x1.c48ddf5beba98p-98"), hf64!("-0x1.39afc8fadbb98p-98"), hf64!("-0x1.37656827f3549p-203")), + (hf64!("-0x1.1d65d5fc31246p-98"), hf64!("-0x1.8ba5360a2b553p-99"), hf64!("-0x1.0bee24969cff9p-204")), + (hf64!("0x1.8a5dd21ef35afp-104"), hf64!("0x1.115aa0fb29a7p-104"), hf64!("-0x1.1p-157")), + (hf64!("0x1.51cc163375e5cp-100"), hf64!("0x1.d4494fb79c5f9p-101"), hf64!("0x1.235a318115a7dp-210")), + (hf64!("0x1.086cbb0e900cfp-96"), hf64!("0x1.6e920d043905ep-97"), hf64!("0x1.0000000000001p-150")), + (hf64!("0x1.f8d2954ab444fp-91"), hf64!("0x1.5dea9642be3ccp-91"), hf64!("0x1.109cd4f731735p-199")), + (hf64!("0x1.8b571f80771bep-90"), hf64!("0x1.12076e976275dp-90"), hf64!("-0x1.fffffffffffffp-144")), + (hf64!("0x1.453f5d718374ap-88"), hf64!("0x1.c2e3888d4911bp-89"), hf64!("0x1.fffffffffffffp-143")), + (hf64!("0x1.101c8b0d7df57p-85"), hf64!("0x1.793a04a8764ap-86"), hf64!("0x1.95385b3628535p-194")), + (hf64!("0x1.99ec5e10fdc8bp-83"), hf64!("0x1.1c231eacb2814p-83"), hf64!("0x1.f82d478ebc7fap-192")), + (hf64!("0x1.530a89e8c2834p-79"), hf64!("0x1.d602c792ff217p-80"), hf64!("0x1.4d765bf788c32p-186")), + (hf64!("0x1.6102ff22f3431p-78"), hf64!("0x1.e960cd938fc71p-79"), hf64!("0x1.fffffffffffffp-133")), + (hf64!("0x1.d7430e6a40aap-78"), hf64!("0x1.46a764f31c407p-78"), hf64!("0x1.fffffffffffffp-132")), + (hf64!("0x1.31677c77b4d02p-62"), hf64!("0x1.a7615378454e9p-63"), hf64!("-0x1.fffffffffffffp-117")), + (hf64!("0x1.760aba9c35ed1p-62"), hf64!("0x1.03441ed22884fp-62"), hf64!("-0x1.fffffffffffffp-116")), + (hf64!("0x1.5f3addb5548c6p-60"), hf64!("0x1.e6e878c6cb8f4p-61"), hf64!("0x1.46dbd6dc7b9eep-168")), + (hf64!("0x1.9a8401f6d9784p-60"), hf64!("0x1.1c8c3a93ce469p-60"), hf64!("-0x1.fffffffffffffp-114")), + (hf64!("0x1.7382bd647eca8p-59"), hf64!("0x1.0182f7f3350e1p-59"), hf64!("0x1.5902345c6d87ep-166")), + (hf64!("0x1.8ddd5485a342fp-56"), hf64!("0x1.13c75940129cbp-56"), hf64!("0x1.365292846d16p-164")), + (hf64!("0x1.2736f975d3b57p-55"), hf64!("0x1.994129328c805p-56"), hf64!("-0x1.fffffffffffffp-110")), + (hf64!("0x1.690ed9e3dd31p-55"), hf64!("0x1.f4885e22dc71bp-56"), hf64!("-0x1.fffffffffffffp-110")), + (hf64!("0x1.e704310d30238p-55"), hf64!("0x1.5192f360cac0ap-55"), hf64!("0x1.30562032a2c95p-161")), + (hf64!("0x1.47bc52cdc6bc1p-53"), hf64!("0x1.c6568b98a9929p-54"), hf64!("0x1.3977497829482p-159")), + (hf64!("0x1.d622a201c030bp-52"), hf64!("0x1.45df797393f11p-52"), hf64!("-0x1.cdcde906092cfp-159")), + (hf64!("0x1.02f371449097fp-51"), hf64!("0x1.66fb73eec9971p-52"), hf64!("0x1.8d182ad6f54d8p-157")), + ]; + + if let Some(result) = lookup_exception(&EXCEPTIONS_TABLE, x) { + return result; + } + + // the 2nd term of the Taylor expansion of 2^x-1 at x=0 is + // log(2)^2/2*x^2 + const C2: f64 = hf64!("0x1.ebfbdff82c58fp-3"); // log(2)^2/2 + + let (hi, lo) = a_mul(LN2H, x); + let lo = LN2L.fma(x, lo); + + // hi+lo approximates the first term x*log(2) + // we add C2*x^2 last, so that in case there is a cancellation in + // LN2L*x+lo, it will contribute more bits + + let lo = lo + (C2 * x * x); + return hi + lo; + } + } + + // now -54 < x < -0x1.0527dbd87e24dp-51 + // or 0x1.0527dbd87e24dp-51 < x < 1024 + + // 2^x-1 is exact for x integer, -53 <= x <= 53 + if ux << 17 == 0 { + cold_path(); + + let i = Float::floor(x); + if x == i && (-53.0..=53.0).contains(&i) { + if i >= 0.0 { + return ((1u64 << i64::cast_from(i)) - 1).cast(); + } else { + return -1.0 + ldexp(1.0, i.cast()); + } + } + } + + let (hi, lo, err) = exp2m1_fast(x, ax <= 0x3fc0000000000000); + let left = hi + (lo - err); + let right = hi + (lo + err); + + if left == right { + return left; + } + + cold_path(); + exp2m1_accurate(x) +} + +/// Given `-54 < x < -0x1.0527dbd87e24dp-51` or `0x1.0527dbd87e24dp-51 < x < 1024`, put in `hi + lo` +/// a double-double approximation of `exp2m1(x)`, and return the maximal corresponding +/// absolute error. +/// The input `tiny` is true iff `|x| <= 0.125`. +#[inline] +fn exp2m1_fast(x: f64, tiny: bool) -> (f64, f64, f64) { + // |x| <= 0.125 + if tiny { + return exp2m1_fast_tiny(x); + } + + // now -54 < x < -0.125 or 0.125 < x < 1024: we approximate exp(x*log(2)) + // and subtract 1 + let (hi, lo) = s_mul(x, LN2H, LN2L); + + // s_mul (hi, lo, x, LN2H, LN2L) decomposes into: + // a_mul (hi, t, x, LN2H) + // lo += fma (x, LN2L, t) + // The a_mul() call is exact, and the error of the fma() is bounded by + // ulp(lo). + // We have |t| <= ulp(hi) <= ulp(LN2H*1024) = 2^-43, + // |t+x*LN2L| <= 2^-43 * 1024*LN2L < 2^-42.7, + // thus |lo| <= |t| + |x*LN2L| + ulp(t+x*LN2L) + // <= 2^-42.7 + 2^-95 <= 2^-42.6, and ulp(lo) <= 2^-95. + // Thus: + // |hi + lo - x*log(2)| <= |hi + lo - x*(LN2H+LN2L)| + |x|*|LN2H+LN2L-log(2)| + // <= 2^-95 + 1024*2^-110.4 < 2^-94.9 + + let (hi, lo) = exp_1(hi, lo); + // hi_out + lo_out = exp(hi + lo) * (1 + eps) with |eps| < 2^-74.139 + // = exp(x*log(2) + eps0) * (1 + eps) + // with |eps0| < 2^-94.9 and |eps| < 2^-74.139 + // = 2^x * (1 + eps1) with |eps1| < 2^-74.138 + + // implies hi >= 1 and the fast_two_sum pre-condition holds + let (hi, u) = if x >= 0.0 { + fast_two_sum(hi, -1.0) + } else { + fast_two_sum(-1.0, hi) // x < 0 thus hi <= 1 + }; + let lo = lo + u; + + // The error in the above fast_two_sum is bounded by 2^-105*|hi|, + // with the new value of hi, thus the total absolute error is bounded + // by eps1*|hi_in|+2^-105*|hi|. + // Relatively to hi this yields eps1*|hi_in/hi| + 2^-105, where the maximum + // of |hi_in/hi| is obtained for x near -0.125, with |2^x/(2^x-1)| < 11.05. + // We get a relative error bound of 2^-74.138*11.05 + 2^-105 < 2^-70.67. + let err = hf64!("0x1.42p-71") * hi; // 2^-70.67 < 0x1.42p-71 + (hi, lo, err) +} + +#[cold] +#[inline] +fn exp2m1_accurate(x: f64) -> f64 { + let ux = x.to_bits(); + let ax = ux & 0x7fffffffffffffff; + + // |x| <= 0.125 + if ax <= 0x3fc0000000000000 { + return exp2m1_accurate_tiny(x); + } + + // now -54 < x < -0.125 or 0.125 < x < 1024: we approximate exp(x*log(2)) + // and subtract 1 + // The following table should be sorted by increasing values of the first + // entry (x). + #[rustfmt::skip] + static EXCEPTIONS_TABLE: [(f64, f64, f64); 93] = [ + (-hf64!("0x1.da22611253866p+0"), -hf64!("0x1.722e3006b8e0bp-1"), -hf64!("0x1.cb1ddc7fbf64ep-109")), + (-hf64!("0x1.9e5bc7d550bb5p+0"), -hf64!("0x1.5943df965427p-1"), hf64!("0x1.f00a26c16d035p-107")), + (-hf64!("0x1.e511f8c5f829fp-1"), -hf64!("0x1.ecfd3f665583fp-2"), hf64!("0x1.cd0996b2d6842p-106")), + (-hf64!("0x1.cef4c143b5adfp-1"), -hf64!("0x1.dcd9f4d61d31fp-2"), hf64!("0x1.88f636e171893p-109")), + (-hf64!("0x1.b444c224a70ccp-1"), -hf64!("0x1.c8b8c01ae382cp-2"), -hf64!("0x1.cb1ddc7fbf64ep-108")), + (-hf64!("0x1.9de261c7c8623p-1"), -hf64!("0x1.b7448eda85236p-2"), hf64!("0x1.e2651db687d79p-106")), + (-hf64!("0x1.750291d22fdap-1"), -hf64!("0x1.95ffcf1509869p-2"), hf64!("0x1.2aabef02ae4cbp-103")), + (-hf64!("0x1.3cb78faaa176ap-1"), -hf64!("0x1.650f7e59509cp-2"), hf64!("0x1.f00a26c16d035p-106")), + (-hf64!("0x1.304447dcfa48ap-1"), -hf64!("0x1.59b94aff7c2b5p-2"), hf64!("0x1.ca6392b99ad78p-106")), + (-hf64!("0x1.e242801b45d0dp-2"), -hf64!("0x1.1d32c9d218361p-2"), hf64!("0x1.03dd41d134594p-109")), + (-hf64!("0x1.d2b12277c95d4p-2"), -hf64!("0x1.155f2f4b7a739p-2"), hf64!("0x1.08b48f6466755p-106")), + (-hf64!("0x1.a4ec34efdd3e1p-2"), -hf64!("0x1.fbc1acbd1558ep-3"), hf64!("0x1.be847d667711dp-105")), + (-hf64!("0x1.8827da1dbf35ep-2"), -hf64!("0x1.dd7867fe8d091p-3"), hf64!("0x1.ffffffffffff8p-57")), + (-hf64!("0x1.75cfa7af70fep-2"), -hf64!("0x1.c9d8c829dad99p-3"), -hf64!("0x1.7e233fcc3fd44p-105")), + (-hf64!("0x1.6d729ff937eadp-2"), -hf64!("0x1.c0d1b0bcb29d6p-3"), hf64!("0x1.7f0f685911fccp-105")), + (-hf64!("0x1.6b3801c55815ap-2"), -hf64!("0x1.be67897b4ab5ep-3"), hf64!("0x1.636be28bf2c88p-106")), + (-hf64!("0x1.6822ccd5b876bp-2"), -hf64!("0x1.bb0f0bff2db03p-3"), hf64!("0x1.4d6bbe2b72be3p-106")), + (-hf64!("0x1.5dbf023667d22p-2"), -hf64!("0x1.afbb4c7215b5bp-3"), hf64!("0x1.3bbc634b006b1p-105")), + (-hf64!("0x1.5a0fba59f338fp-2"), -hf64!("0x1.abb2028f6491cp-3"), hf64!("0x1.0a9c5a206828ep-104")), + (-hf64!("0x1.540658c7eedcap-2"), -hf64!("0x1.a50f96bbf778cp-3"), hf64!("0x1.f14fe3c42820cp-105")), + (-hf64!("0x1.31c2bbece259bp-2"), -hf64!("0x1.7ee2b68e61af7p-3"), -hf64!("0x1.7852d3ddee861p-107")), + (-hf64!("0x1.253dd048bf2e7p-2"), -hf64!("0x1.70b709e65877ep-3"), hf64!("0x1.3625f20d5b638p-104")), + (-hf64!("0x1.1249283f53506p-2"), -hf64!("0x1.5b076e9ede20bp-3"), hf64!("0x1.14d2b0b9298b2p-105")), + (-hf64!("0x1.0f2086c7b8f83p-2"), -hf64!("0x1.57636305fd889p-3"), hf64!("0x1.9442185db9d73p-104")), + (-hf64!("0x1.ff6ff72ac299dp-3"), -hf64!("0x1.458423bb5b025p-3"), hf64!("0x1.140aa7335a14p-104")), + (-hf64!("0x1.fe89353e31cbfp-3"), -hf64!("0x1.44fd97cc9857p-3"), -hf64!("0x1.a549adf007feap-109")), + (-hf64!("0x1.ec7298061ebfap-3"), -hf64!("0x1.3a68f321de8abp-3"), hf64!("0x1.7f700400ca84fp-103")), + (-hf64!("0x1.d4854d9f87fcap-3"), -hf64!("0x1.2c506fcda97fcp-3"), -hf64!("0x1.a4fedf7b3ca3fp-109")), + (-hf64!("0x1.a81224ff9a5cfp-3"), -hf64!("0x1.11d2b7526eae5p-3"), hf64!("0x1.6fb9bdbb725b2p-106")), + (-hf64!("0x1.93314fd04393cp-3"), -hf64!("0x1.053dee1b48ebdp-3"), hf64!("0x1.2c15fbfa775b3p-104")), + (-hf64!("0x1.8cd71c092c54dp-3"), -hf64!("0x1.0165759fe661fp-3"), hf64!("0x1.395807142ed1ep-104")), + (-hf64!("0x1.747fb97a9be61p-3"), -hf64!("0x1.e52ae5003c9dbp-4"), -hf64!("0x1.dbf62ab30e6cp-106")), + (-hf64!("0x1.64b0e525bc004p-3"), -hf64!("0x1.d1cc0e1d08fa9p-4"), hf64!("0x1.69442904d7698p-104")), + (-hf64!("0x1.539353db1bdebp-3"), -hf64!("0x1.bcb50aacb6649p-4"), -hf64!("0x1.a8b1eddf109b5p-107")), + (-hf64!("0x1.4faff91bda7ddp-3"), -hf64!("0x1.b7e630a3fab2ap-4"), hf64!("0x1.47706b5ad802ap-105")), + (-hf64!("0x1.4c2bec34d5837p-3"), -hf64!("0x1.b38bcbad2b407p-4"), -hf64!("0x1.b4e03df36240cp-108")), + (-hf64!("0x1.439bb08833472p-3"), -hf64!("0x1.a8ebef4998a79p-4"), -hf64!("0x1.2c3a2b30f99cfp-105")), + (-hf64!("0x1.418f83945849ap-3"), -hf64!("0x1.a6606e818c2e3p-4"), hf64!("0x1.145b99cfb763cp-104")), + (-hf64!("0x1.38417d7db5a2fp-3"), -hf64!("0x1.9aca18a87c03ap-4"), hf64!("0x1.76a10886fcc65p-105")), + (-hf64!("0x1.3041fe0fc550bp-3"), -hf64!("0x1.90ccdeae8d3ddp-4"), hf64!("0x1.f202cb751932p-104")), + (-hf64!("0x1.236e786c083dbp-3"), -hf64!("0x1.80b96ddbdb1b3p-4"), hf64!("0x1.ec381ad5e1718p-105")), + (-hf64!("0x1.06c0eba373a18p-3"), -hf64!("0x1.5c86ed4f55301p-4"), hf64!("0x1.fffffffffffe3p-58")), + (-hf64!("0x1.06559d8a9ff1ap-3"), -hf64!("0x1.5bfed18cc8ff3p-4"), hf64!("0x1.87a53f06d41bdp-105")), + (-hf64!("0x1.0006ec0817742p-3"), -hf64!("0x1.53fc5e2bd9f6cp-4"), -hf64!("0x1.6fcda9953133bp-107")), + (hf64!("0x1.06d6b01577482p-3"), hf64!("0x1.7d11edfaac429p-4"), hf64!("0x1.ffffffffffffbp-58")), + (hf64!("0x1.079724fbb3d86p-3"), hf64!("0x1.7e35970e11732p-4"), -hf64!("0x1.b3086fea6e73cp-107")), + (hf64!("0x1.0e2f828c93192p-3"), hf64!("0x1.88376cd2f4076p-4"), -hf64!("0x1.0bc0000d21ae1p-106")), + (hf64!("0x1.111c072cd8f6ep-3"), hf64!("0x1.8ca9045ba3e95p-4"), hf64!("0x1.80f80d8a8722ap-105")), + (hf64!("0x1.165c1c6243beep-3"), hf64!("0x1.94a6924c0b159p-4"), hf64!("0x1.ffffffffffff9p-58")), + (hf64!("0x1.182f8cb329d34p-3"), hf64!("0x1.976ed12501a66p-4"), -hf64!("0x1.b0e5142b2ec14p-105")), + (hf64!("0x1.1a8801599f756p-3"), hf64!("0x1.9b02639918dcdp-4"), hf64!("0x1.736bd9c878fbap-107")), + (hf64!("0x1.1bb1b31cf8c98p-3"), hf64!("0x1.9cc894e900c64p-4"), hf64!("0x1.218cf2d9f3926p-104")), + (hf64!("0x1.2ba99934336fcp-3"), hf64!("0x1.b5379b9a6155cp-4"), hf64!("0x1.68cdeb7f25f81p-106")), + (hf64!("0x1.2dbe99fec947ap-3"), hf64!("0x1.b869aac963d94p-4"), hf64!("0x1.59f0621f5fc6ep-107")), + (hf64!("0x1.2eecf76d63cdp-3"), hf64!("0x1.ba39ff28e3eap-4"), -hf64!("0x1.cb1ddc7fbf64ep-107")), + (hf64!("0x1.45c447c5552ddp-3"), hf64!("0x1.dd721415a4a8dp-4"), -hf64!("0x1.e5111c4062fa9p-108")), + (hf64!("0x1.502f26bc1aa18p-3"), hf64!("0x1.ed99608511a69p-4"), -hf64!("0x1.603952f77b6d4p-106")), + (hf64!("0x1.5856bdae2f08fp-3"), hf64!("0x1.fa489ed69464fp-4"), hf64!("0x1.93f08a7946826p-104")), + (hf64!("0x1.5f38bbc1a9944p-3"), hf64!("0x1.028230c4e5389p-3"), hf64!("0x1.313458d3f84a2p-105")), + (hf64!("0x1.691b76c4b3c1bp-3"), hf64!("0x1.0a3d166eacab7p-3"), hf64!("0x1.9a98ee8a745ffp-104")), + (hf64!("0x1.6c4175ea0c6e1p-3"), hf64!("0x1.0cb4b9c7701a4p-3"), hf64!("0x1.a6605f3be564ep-108")), + (hf64!("0x1.6f6544ef96539p-3"), hf64!("0x1.0f2b51ddc90e1p-3"), hf64!("0x1.06e9b6d330daep-105")), + (hf64!("0x1.7a3ab625d0dc4p-3"), hf64!("0x1.17b04242e48fep-3"), hf64!("0x1.60215b123bdbdp-108")), + (hf64!("0x1.7d33f0271269bp-3"), hf64!("0x1.1a0842d5b11e4p-3"), -hf64!("0x1.5c29b4c0046f9p-106")), + (hf64!("0x1.887678e0de774p-3"), hf64!("0x1.22edc495eb728p-3"), hf64!("0x1.e2651db687d79p-105")), + (hf64!("0x1.8fe3adec251f6p-3"), hf64!("0x1.28d0ae0c0ac4fp-3"), hf64!("0x1.e1bce5201668bp-104")), + (hf64!("0x1.926961243babap-3"), hf64!("0x1.2ad15442773c8p-3"), hf64!("0x1.112edd525e11cp-105")), + (hf64!("0x1.a066bd757bf54p-3"), hf64!("0x1.35f4af72a0a25p-3"), -hf64!("0x1.59b2d13ebb9d3p-106")), + (hf64!("0x1.b00f80708d157p-3"), hf64!("0x1.427c60cc0d217p-3"), hf64!("0x1.344b77789552ap-103")), + (hf64!("0x1.b8ee8dcb0e797p-3"), hf64!("0x1.499d2a386003fp-3"), -hf64!("0x1.e5d43d3c21c2ap-105")), + (hf64!("0x1.c3b0f79893bfcp-3"), hf64!("0x1.5249b249243b7p-3"), hf64!("0x1.1a9819f194524p-105")), + (hf64!("0x1.d5ac74e107b39p-3"), hf64!("0x1.60db0007c78ffp-3"), -hf64!("0x1.a936bca36028fp-106")), + (hf64!("0x1.e9532c87abc5bp-3"), hf64!("0x1.70e070f8d9706p-3"), -hf64!("0x1.0ab97f38bd529p-106")), + (hf64!("0x1.16383a5fe20dp-2"), hf64!("0x1.a867756a9a8fep-3"), -hf64!("0x1.8ac73f07192e6p-106")), + (hf64!("0x1.2a381dbbc8f07p-2"), hf64!("0x1.ca1a3ba1db0ebp-3"), -hf64!("0x1.7f5a08441b722p-105")), + (hf64!("0x1.8690e0aabd12cp-2"), hf64!("0x1.35e1034d5ec8p-2"), hf64!("0x1.f00a26c16d035p-105")), + (hf64!("0x1.a246f399a957bp-2"), hf64!("0x1.4f22bccd234f1p-2"), hf64!("0x1.ffffffffffff6p-56")), + (hf64!("0x1.b808c857723e7p-2"), hf64!("0x1.634ce84a0e79dp-2"), hf64!("0x1.ffffffffffff4p-56")), + (hf64!("0x1.bb2446a7b770fp-2"), hf64!("0x1.663466e4bb92bp-2"), -hf64!("0x1.a00681268d32cp-106")), + (hf64!("0x1.c14d1ce57d7ebp-2"), hf64!("0x1.6bfacbdf4334ep-2"), hf64!("0x1.1cfc31c20bd19p-104")), + (hf64!("0x1.f8432a3c6c453p-2"), hf64!("0x1.a096f253eca4bp-2"), hf64!("0x1.56c0f7fb6fa16p-106")), + (hf64!("0x1.16a76ec41b516p-1"), hf64!("0x1.d541a1690b18ep-2"), hf64!("0x1.08b48f6466755p-105")), + (hf64!("0x1.1e5f48d8ba05ap-1"), hf64!("0x1.e4f0e78affdcdp-2"), hf64!("0x1.ffffffffffffep-56")), + (hf64!("0x1.4a63ff1d53f53p-1"), hf64!("0x1.20cc3b425aa51p-1"), hf64!("0x1.636be28bf2c88p-105")), + (hf64!("0x1.d906bc1caeb3dp-1"), hf64!("0x1.cb5f21a856e8dp-1"), hf64!("0x1.c737d097094f7p-104")), + (hf64!("0x1.25dd9eedac79ap+0"), hf64!("0x1.37473fe51c7d4p+0"), -hf64!("0x1.cb1ddc7fbf64ep-106")), + (hf64!("0x1.61a4382aaf44bp+0"), hf64!("0x1.9af081a6af64p+0"), hf64!("0x1.f00a26c16d035p-104")), + (hf64!("0x1.8b53b7620da8bp+0"), hf64!("0x1.eaa0d0b4858c7p+0"), hf64!("0x1.08b48f6466755p-104")), + (hf64!("0x1.12eecf76d63cdp+1"), hf64!("0x1.b7473fe51c7d4p+1"), -hf64!("0x1.cb1ddc7fbf64ep-105")), + (hf64!("0x1.92eecf76d63cdp+1"), hf64!("0x1.f7473fe51c7d4p+2"), -hf64!("0x1.cb1ddc7fbf64ep-104")), + (hf64!("0x1.ffef3f31766b7p+5"), hf64!("0x1.fd1ae8d0e6abdp+63"), -hf64!("0x1.5ea4d3ad23ccep-42")), + (hf64!("0x1.586b62dc1e8c2p+6"), hf64!("0x1.134d37339fe93p+86"), -hf64!("0x1.1295610d3deffp-20")), + (hf64!("0x1.6f33cdaf56d6p+6"), hf64!("0x1.bde76c2db52fbp+91"), hf64!("0x1.564254439c8a9p-15")), + ]; + + if let Some(result) = lookup_exception(&EXCEPTIONS_TABLE, x) { + return result; + } + + let (hi, lo) = exp_2(x); + + // implies hi >= 1 and the fast_two_sum pre-condition holds + let (hi, u) = if x >= 0.0 { + fast_two_sum(hi, -1.0) + } else { + fast_two_sum(-1.0, hi) // x < 0 thus hi <= 1 + }; + + let lo = lo + u; + + // the error in the above fast_two_sum is bounded by 2^-105*|hi|, + // with the new value of hi + return hi + lo; +} + +/// The following is a degree-10 polynomial generated by Sollya (file `exp2m1_fast.sollya`), +/// which approximates `exp2m1(x)` with relative error bounded by `2^-68.559` for `|x| <= 0.125`. +const P: [f64; 12] = [ + hf64!("0x1.62e42fefa39efp-1"), + hf64!("0x1.abd1697afcaf8p-56"), // degree 1, P[0], P[1] + hf64!("0x1.ebfbdff82c58fp-3"), + -hf64!("0x1.5e5a1d09e1599p-57"), // degree 2, P[2], P[3] + hf64!("0x1.c6b08d704a0bfp-5"), // degree 3, P[4] + hf64!("0x1.3b2ab6fba4e78p-7"), // degree 4, P[5] + hf64!("0x1.5d87fe78a84e6p-10"), // degree 5, P[6] + hf64!("0x1.430912f86a48p-13"), // degree 6, P[7] + hf64!("0x1.ffcbfbc1f2b36p-17"), // degree 7, P[8] + hf64!("0x1.62c0226c7f6d1p-20"), // degree 8, P[9] + hf64!("0x1.b539529819e63p-24"), // degree 9, P[10] + hf64!("0x1.e4d552bed5b9cp-28"), // degree 10, P[11] +]; + +/// `|x| <= 0.125`, put in `hi + lo` a double-double approximation of `exp2m1(x)`, and return the +/// maximal corresponding absolute error. We also have `|x| > 0x1.0527dbd87e24dp-51`. +/// With `xmin=RR("0x1.0527dbd87e24dp-51",16)`, the routine +/// `exp2m1_fast_tiny_all(xmin,0.125,2^-65.73)` in `exp2m1.sage` returns +/// `1.63414352331297e-20 < 2^-65.73`, and `exp2m1_fast_tiny_all(-0.125,-xmin,2^-65.62)` returns +/// `1.76283772822891e-20 < 2^-65.62`, which proves the relative error is bounded by `2^-65.62`. +#[inline] +fn exp2m1_fast_tiny(x: f64) -> (f64, f64, f64) { + // The maximal value of |c4*x^4/exp2m1(x)| over [-0.125,0.125] + // is less than 2^-15.109, where c4 is the degree-4 coefficient, + // thus we can compute the coefficients of degree 4 or higher + // using double precision only. + let x2 = x * x; + let x4 = x2 * x2; + let c8 = P[10].fma(x, P[9]); // degree 8 + let c6 = P[8].fma(x, P[7]); // degree 6 + let c4 = P[6].fma(x, P[5]); // degree 4 + let c8 = P[11].fma(x2, c8); // degree 8 + let c4 = c6.fma(x2, c4); // degree 4 + let c4 = c8.fma(x4, c4); // degree 4 + + // multiply c4 by x and add P[4] + let (hi, lo) = a_mul(c4, x); + let (hi, lo) = fast_sum(P[4], hi, lo); + + // multiply (hi,lo) by x and add P[2]+P[3] + let (hi, lo) = s_mul(x, hi, lo); + let (hi, t) = fast_two_sum(P[2], hi); + let lo = lo + (t + P[3]); + + // multiply (hi,lo) by x and add P[0]+P[1] + let (hi, lo) = s_mul(x, hi, lo); + let (hi, t) = fast_two_sum(P[0], hi); + let lo = lo + (t + P[1]); + + // multiply (hi,lo) by x + let (hi, lo) = s_mul(x, hi, lo); + let err = hf64!("0x1.4ep-66") * hi; // 2^-65.62 < 0x1.4ep-66 + (hi, lo, err) +} + +// The following is a degree-15 polynomial generated by Sollya +// (file exp2m1_accurate.sollya), +// which approximates exp2m1(x) with relative error bounded by 2^-107.666 +// for |x| <= 0.125. +const Q: [f64; 22] = [ + hf64!("0x1.62e42fefa39efp-1"), + hf64!("0x1.abc9e3b39804p-56"), // degree 1: Q[0], Q[1] + hf64!("0x1.ebfbdff82c58fp-3"), + -hf64!("0x1.5e43a53e44dcfp-57"), // degree 2: Q[2], Q[3] + hf64!("0x1.c6b08d704a0cp-5"), + -hf64!("0x1.d331627517168p-59"), // degree 3: Q[4], Q[5] + hf64!("0x1.3b2ab6fba4e77p-7"), + hf64!("0x1.4e65df0779f8cp-62"), // degree 4: Q[6], Q[7] + hf64!("0x1.5d87fe78a6731p-10"), + hf64!("0x1.0717fbf4bd05p-66"), // degree 5: Q[8], Q[9] + hf64!("0x1.430912f86c787p-13"), + hf64!("0x1.bd2bdec9bcd42p-67"), // degree 6: Q[10], Q[11] + hf64!("0x1.ffcbfc588b0c7p-17"), + -hf64!("0x1.e60aa6d5e4aa9p-71"), // degree 7: Q[12], Q[13] + hf64!("0x1.62c0223a5c824p-20"), // degree 8: Q[14] + hf64!("0x1.b5253d395e7d4p-24"), // degree 9: Q[15] + hf64!("0x1.e4cf5158b916p-28"), // degree 10: Q[16] + hf64!("0x1.e8cac734c6058p-32"), // degree 11: Q[17] + hf64!("0x1.c3bd64f17199dp-36"), // degree 12: Q[18] + hf64!("0x1.8161a17e05651p-40"), // degree 13: Q[19] + hf64!("0x1.3150b3d792231p-44"), // degree 14: Q[20] + hf64!("0x1.c184260bfad7ep-49"), // degree 15: Q[21] +]; + +/// Accurate path for `0x1.0527dbd87e24dp-51 < |x| <= 0.125`. +#[inline] +fn exp2m1_accurate_tiny(x: f64) -> f64 { + // The following exceptional cases have at least 51 identical bits after + // the round bit, thus are hard to correctly round with double-double + // arithmetic. They should be sorted by increasing values of the first + // entry (x). + #[rustfmt::skip] + static EXCEPTIONS_TABLE: [(f64, f64, f64); 59] = [ + (-hf64!("0x1.f6ec73d3948c3p-4"), -hf64!("0x1.4e2d8b0cead45p-4"), -hf64!("0x1.40f3d5244acffp-109")), + (-hf64!("0x1.9b28778a9a8a3p-4"), -hf64!("0x1.134dec3a6324bp-4"), -hf64!("0x1.1acaaac7e1ff7p-110")), + (-hf64!("0x1.6f94484e5e1fdp-5"), -hf64!("0x1.f5baee010ccc6p-6"), -hf64!("0x1.3298bcde4f9a8p-115")), + (-hf64!("0x1.3918e8608bd5bp-8"), -hf64!("0x1.b153bf52832f9p-9"), -hf64!("0x1.fffffffffffffp-63")), + (-hf64!("0x1.8474969f5eb14p-9"), -hf64!("0x1.0cfafc07a957bp-9"), -hf64!("0x1.fffffffffffffp-63")), + (-hf64!("0x1.0867d8153350dp-9"), -hf64!("0x1.6e49b44e387f5p-10"), -hf64!("0x1.fffffffffffffp-64")), + (-hf64!("0x1.bfb3efcdf2bc4p-10"), -hf64!("0x1.3623eff91de91p-10"), hf64!("0x1.cd506bd8d0439p-117")), + (-hf64!("0x1.0c3ebbd1a501fp-11"), -hf64!("0x1.73ccf8ee62819p-12"), hf64!("0x1.3a3965a926c6dp-120")), + (-hf64!("0x1.3c7971b0ee205p-14"), -hf64!("0x1.b6b716c4bb87cp-15"), hf64!("0x1.d7150e84d973dp-121")), + (-hf64!("0x1.3dbf41403c0b2p-15"), -hf64!("0x1.b87c37192e4f7p-16"), hf64!("0x1.098861b427b13p-123")), + (-hf64!("0x1.3b6786a5a9a69p-17"), -hf64!("0x1.b53dee1a96ca4p-18"), -hf64!("0x1.282e0781f97f5p-124")), + (-hf64!("0x1.7b388eb924102p-18"), -hf64!("0x1.06dafba21afffp-18"), hf64!("0x1.fffffffffffffp-72")), + (-hf64!("0x1.98aae9950914ep-19"), -hf64!("0x1.1b443a4805f8fp-19"), hf64!("0x1.fffffffffffffp-73")), + (-hf64!("0x1.41e4bec9bc547p-19"), -hf64!("0x1.be3d238468f35p-20"), -hf64!("0x1.fffffffffffffp-74")), + (-hf64!("0x1.0f1c08e43f217p-20"), -hf64!("0x1.77d66368613b4p-21"), hf64!("0x1.0d3de71c269fcp-130")), + (-hf64!("0x1.0d296993d1368p-20"), -hf64!("0x1.752326c780f68p-21"), hf64!("0x1.1cade8dbbd5d8p-130")), + (-hf64!("0x1.8288f6bb77d79p-23"), -hf64!("0x1.0becf6ad924bfp-23"), -hf64!("0x1.fffffffffffffp-77")), + (-hf64!("0x1.0ee8225c19555p-23"), -hf64!("0x1.778e77e8b1e13p-24"), -hf64!("0x1.3da14df43e675p-129")), + (-hf64!("0x1.922076a30742p-24"), -hf64!("0x1.16bba98a001c1p-24"), hf64!("0x1.d90eddeada3e4p-133")), + (-hf64!("0x1.3b6203ff2cbe6p-29"), -hf64!("0x1.b536a7dace8cap-30"), hf64!("0x1.07bb47158694fp-137")), + (-hf64!("0x1.e07a2fbc7fd84p-30"), -hf64!("0x1.4d0a9e634dc35p-30"), -hf64!("0x1.fffffffffffffp-84")), + (-hf64!("0x1.ba24ff5dea796p-34"), -hf64!("0x1.3278a26ed0162p-34"), hf64!("0x1.f837135341102p-142")), + (-hf64!("0x1.d091d774b141ep-35"), -hf64!("0x1.4203e2685b069p-35"), -hf64!("0x1.a166cdf05ff79p-143")), + (-hf64!("0x1.bfc1e9b0f73aep-35"), -hf64!("0x1.365ca0d933491p-35"), -hf64!("0x1.fffffffffffffp-89")), + (-hf64!("0x1.6d752e77a9dc2p-41"), -hf64!("0x1.faa1cb0d787b3p-42"), -hf64!("0x1.fffffffffffffp-96")), + (-hf64!("0x1.a1f242e670d73p-42"), -hf64!("0x1.21b2c54479c4dp-42"), hf64!("0x1.48a5d173822e8p-150")), + (-hf64!("0x1.66ff2474821a7p-44"), -hf64!("0x1.f1accede78f51p-45"), -hf64!("0x1.fffffffffffffp-99")), + (-hf64!("0x1.3f596b7b1c5e2p-44"), -hf64!("0x1.bab64e105226ap-45"), -hf64!("0x1.1a2829f825272p-154")), + (-hf64!("0x1.daaead688f65fp-45"), -hf64!("0x1.4906541fb8d5cp-45"), -hf64!("0x1.31de54e6dc4ep-151")), + (-hf64!("0x1.0e5678a7b8bap-47"), -hf64!("0x1.76c48a7a527bp-48"), -hf64!("0x1.6fbb13d1e3faap-155")), + (-hf64!("0x1.a6d6a49f2187fp-50"), -hf64!("0x1.2516dafdf17adp-50"), -hf64!("0x1.85a87dc0b88p-157")), + (-hf64!("0x1.2716da024f6d9p-50"), -hf64!("0x1.9914a112c8dadp-51"), hf64!("0x1.004287d8393fep-159")), + (-hf64!("0x1.2c506b0368099p-51"), -hf64!("0x1.a052e3d5e791p-52"), -hf64!("0x1.428e13d1da7cap-159")), + (hf64!("0x1.391609b20beaap-51"), hf64!("0x1.b2078ba8f6835p-52"), -hf64!("0x1.85099d92fa919p-162")), + (hf64!("0x1.4bbbd21c8c721p-51"), hf64!("0x1.cbe169f09f95ep-52"), -hf64!("0x1.256d63281c5efp-158")), + (hf64!("0x1.29d3990338b4ep-50"), hf64!("0x1.9ce011cf5f46ep-51"), -hf64!("0x1.ba552cb13d27bp-161")), + (hf64!("0x1.d57070df38af8p-50"), hf64!("0x1.4563f61023f7p-50"), -hf64!("0x1.ac66c9a5a19p-157")), + (hf64!("0x1.f6ac7a2928816p-49"), hf64!("0x1.5c6d4854f6b3bp-49"), hf64!("0x1.fffffffffffffp-103")), + (hf64!("0x1.c2ff91c241999p-47"), hf64!("0x1.389bb3cfc42c2p-47"), -hf64!("0x1.023c49dbbc653p-154")), + (hf64!("0x1.ac65fdb418f72p-40"), hf64!("0x1.28f171f05f6fap-40"), -hf64!("0x1.51612679f6f55p-146")), + (hf64!("0x1.f06778562781fp-38"), hf64!("0x1.5814c6c101c7p-38"), -hf64!("0x1.333ad12f679a5p-145")), + (hf64!("0x1.02cdc0b0f314cp-37"), hf64!("0x1.66c7342a5b89fp-38"), -hf64!("0x1.fffffffffffffp-92")), + (hf64!("0x1.2162f23d082d5p-36"), hf64!("0x1.912cc5483c4bbp-37"), -hf64!("0x1.9b93a2f7e5da4p-145")), + (hf64!("0x1.2d664653786ecp-36"), hf64!("0x1.a1d414c603907p-37"), hf64!("0x1.fffffffffffffp-91")), + (hf64!("0x1.b3665a468351p-35"), hf64!("0x1.2dcbd0c2b922fp-35"), -hf64!("0x1.fffffffffffffp-89")), + (hf64!("0x1.3dc2574084694p-34"), hf64!("0x1.b881f93d3d7bcp-35"), hf64!("0x1.4d32033399826p-141")), + (hf64!("0x1.b988be78bfb6bp-33"), hf64!("0x1.320c53ed5fb63p-33"), -hf64!("0x1.fffffffffffffp-87")), + (hf64!("0x1.e755f4897c124p-27"), hf64!("0x1.51cba0164bed2p-27"), -hf64!("0x1.16a01564a1359p-134")), + (hf64!("0x1.e4c5c9414182ep-26"), hf64!("0x1.5004cdd3ec269p-26"), -hf64!("0x1.411c828b7c78ep-133")), + (hf64!("0x1.1c4baf7475e2dp-23"), hf64!("0x1.8a1e1272b4997p-24"), hf64!("0x1.fffffffffffffp-78")), + (hf64!("0x1.3a6de04d8d384p-22"), hf64!("0x1.b3e437cdb56dep-23"), hf64!("0x1p-76")), + (hf64!("0x1.4753a08baf7fbp-22"), hf64!("0x1.c5c56aeb3c161p-23"), hf64!("0x1.fffffffffffffp-77")), + (hf64!("0x1.b6275c496a3d5p-16"), hf64!("0x1.2fb5318232c0cp-16"), hf64!("0x1.666c5e4de0763p-123")), + (hf64!("0x1.02df58d57f81fp-13"), hf64!("0x1.66e3866f48983p-14"), -hf64!("0x1.3e42ea0bb96a5p-121")), + (hf64!("0x1.a949818632705p-13"), hf64!("0x1.26ceaaf78e39fp-13"), -hf64!("0x1.fffffffffffffp-67")), + (hf64!("0x1.dcc98bad34bd4p-13"), hf64!("0x1.4a82829ba33efp-13"), hf64!("0x1.3a222409353fep-120")), + (hf64!("0x1.8ff59817c7989p-9"), hf64!("0x1.15862abd340e8p-9"), hf64!("0x1.6ab813a41d504p-116")), + (hf64!("0x1.6a8ee6a3521fdp-6"), hf64!("0x1.fa7ca1e176885p-7"), hf64!("0x1.6530cb0e0ba64p-113")), + (hf64!("0x1.d4a8ebce833a3p-4"), hf64!("0x1.52145769df4fbp-4"), -hf64!("0x1.5d8026da33cabp-111")), + ]; + + if let Some(result) = lookup_exception(&EXCEPTIONS_TABLE, x) { + return result; + } + + let x2 = x * x; + let x4 = x2 * x2; + + let c13 = Q[20].fma(x, Q[19]); // degree 13 + let c11 = Q[18].fma(x, Q[17]); // degree 11 + let c13 = Q[21].fma(x2, c13); // degree 13 + + // add Q[16]*x+c11*x2+c13*x4 to Q[15] (degree 9) + let (hi, lo) = fast_two_sum(Q[15], Q[16] * x + c11 * x2 + c13 * x4); + // multiply hi+lo by x and add Q[14] (degree 8) + let (hi, lo) = s_mul(x, hi, lo); + let (hi, t) = fast_two_sum(Q[14], hi); + let lo = lo + t; + + // multiply hi+lo by x and add Q[12]+Q[13] (degree 7) + let (hi, lo) = s_mul(x, hi, lo); + let (hi, t) = fast_two_sum(Q[12], hi); + let lo = lo + (t + Q[13]); + + // multiply hi+lo by x and add Q[10]+Q[11] (degree 6) + let (hi, lo) = s_mul(x, hi, lo); + let (hi, t) = fast_two_sum(Q[10], hi); + let lo = lo + (t + Q[11]); + + // multiply hi+lo by x and add Q[8]+Q[9] (degree 5) + let (hi, lo) = s_mul(x, hi, lo); + let (hi, t) = fast_two_sum(Q[8], hi); + let lo = lo + (t + Q[9]); + + // multiply hi+lo by x and add Q[6]+Q[7] (degree 4) + let (hi, lo) = s_mul(x, hi, lo); + let (hi, t) = fast_two_sum(Q[6], hi); + let lo = lo + (t + Q[7]); + + // multiply hi+lo by x and add Q[4]+Q[5] (degree 3) + let (hi, lo) = s_mul(x, hi, lo); + let (hi, t) = fast_two_sum(Q[4], hi); + let lo = lo + (t + Q[5]); + + // multiply hi+lo by x and add Q[2]+Q[3] (degree 2) + let (hi, lo) = s_mul(x, hi, lo); + let (hi, t) = fast_two_sum(Q[2], hi); + let lo = lo + (t + Q[3]); + + // multiply hi+lo by x and add Q[0]+Q[1] (degree 2) + let (hi, lo) = s_mul(x, hi, lo); + let (hi, t) = fast_two_sum(Q[0], hi); + let lo = lo + (t + Q[1]); + + // multiply hi+lo by x + let (hi, lo) = s_mul(x, hi, lo); + + hi + lo +} + +/// Approximation of `exp(x)`, where `x = xh + xl` +/// +/// `exp(x)` is approximated by `hi + lo`. +/// +/// For the error analysis, we only consider the case where `x^y` does not overflow or underflow. +/// We get: +/// +/// `(hi + lo) / exp(xh + xl) = 1 + eps` with `|eps| < 2^-74.139` +/// +/// Assumes `|xl/xh| < 2^-23.89` and `|xl| < 2^-14.3486`. +/// +/// See analysis before the `exp_1()` call in `cr_pow()`, which proves +/// `|rl| < 2^-23.89 |rh|` (here `xh=rh` and `xl=rl`). +/// +/// At output, we also have `0.99985 < hi+lo < 1.99995` and `|lo/hi| < 2^-41.4`. +#[inline] +fn exp_1(xh: f64, xl: f64) -> (f64, f64) { + // |INVLOG2-2^12/log(2)| < 2^-43.4 + const INVLOG2: f64 = hf64!("0x1.71547652b82fep+12"); + const LOG2H: f64 = hf64!("0x1.62e42fefa39efp-13"); + const LOG2L: f64 = hf64!("0x1.abc9e3b39803fp-68"); + + let k = (xh * INVLOG2).roundeven(); + let (kh, kl) = s_mul(k, LOG2H, LOG2L); + let (yh, yl) = fast_two_sum(xh - kh, xl); + let yl = yl - kl; + + // Note: k is an integer, this is just a conversion. + let k = i64::cast_from(k); + let m = (k >> 12) + 0x3ff; + let i2 = usize::cast_from((k >> 6) & 0x3f); + let i1 = usize::cast_from(k & 0x3f); + + let (t1h, t1l) = T1[i2]; + let (t2h, t2l) = T2[i1]; + let (hi, lo) = d_mul(t2h, t2l, t1h, t1l); + let (qh, ql) = q_1(yh, yl); + let (mut hi, mut lo) = d_mul(hi, lo, qh, ql); + + // Scale by 2^k. Warning: for x near 1024, we can have k=2^22, thus + // m = 2047, which encodes Inf + let mut scale = f64::from_bits(m.unsigned() << 52); + if m == 0x7ff { + cold_path(); + hi *= 2.0; + lo *= 2.0; + scale = f64::from_bits((m - 1).unsigned() << 52); + } + hi *= scale; + lo *= scale; + (hi, lo) +} + +/// returns a double-double approximation `hi+lo` of `exp(x*log(2))` for `|x| < 745` +#[inline] +fn exp_2(x: f64) -> (f64, f64) { + let k = (x * hf64!("0x1p12")).roundeven(); + // since |x| <= 745 we have k <= 3051520 + let yh = (-k).fma(hf64!("0x1p-12"), x); // exact, |yh| <= 2^-13 + // now x = k + yh, thus 2^x = 2^k * 2^yh, and we multiply yh by log(2) + // to use the accurate path of exp() + let (yh, yl) = s_mul(yh, LN2H, LN2L); + + // Note: k is an integer, this is just a conversion. + let k = i64::cast_from(k); + let m = (k >> 12) + 0x3ff; + let i2 = usize::cast_from((k >> 6) & 0x3f); + let i1 = usize::cast_from(k & 0x3f); + + let (t1h, t1l) = T1[i2]; + let (t2h, t2l) = T2[i1]; + let (hi, lo) = d_mul(t2h, t2l, t1h, t1l); + let (qh, ql) = q_2(yh, yl); + let (mut hi, mut lo) = d_mul(hi, lo, qh, ql); + + // Scale by 2^k. Warning: for x near 1024, we can have k=2^22, thus + // m = 2047, which encodes Inf + let mut scale = f64::from_bits(m.unsigned() << 52); + if m == 0x7ff { + cold_path(); + hi *= 2.0; + lo *= 2.0; + scale = f64::from_bits((m - 1).unsigned() << 52); + } + hi *= scale; + lo *= scale; + (hi, lo) +} + +/// The following is a degree-4 polynomial generated by Sollya for `exp(x)` over +/// `[-0.000130273,0.000130273]` with absolute error `< 2^-74.346`. +const Q_1: [f64; 5] = [ + hf64!("0x1p0"), // degree 0 + hf64!("0x1p0"), // degree 1 + hf64!("0x1p-1"), // degree 2 + hf64!("0x1.5555555995d37p-3"), // degree 3 + hf64!("0x1.55555558489dcp-5"), // degree 4 +]; + +/// Approximation for the fast path of `exp(z)` for `z=zh+zl`, with +/// `|z| < 0.000130273 < 2^-12.88` and `|zl| < 2^-42.6` (assuming `x^y` does not overflow +/// or underflow) +#[inline] +fn q_1(zh: f64, zl: f64) -> (f64, f64) { + let z = zh + zl; + let q = Q_1[4].fma(zh, Q_1[3]); + let q = q.fma(z, Q_1[2]); + let (hi, lo) = fast_two_sum(Q_1[1], q * z); + let (hi, lo) = d_mul(zh, zl, hi, lo); + fast_sum(Q_1[0], hi, lo) +} + +// FIXME(expm1): use this directly from `expm1` once ported from CORE-MATH. + +/// The following is a degree-7 polynomial generated by Sollya for `exp(z)` over +/// `[-0.000130273,0.000130273]` with absolute error `< 2^-113.218` +/// (see file `exp_accurate.sollya` in the CORE-MATH repo). Since we use this code only for +/// `|x| > 0.125` in `exp2m1(x)`, the corresponding relative error for `exp2m1` is about +/// `2^-113.218/|exp2m1(-0.125)|` which is about `2^-110`. +const Q_2: [f64; 9] = [ + hf64!("0x1p0"), // degree 0, Q_2[0] + hf64!("0x1p0"), // degree 1, Q_2[1] + hf64!("0x1p-1"), // degree 2, Q_2[2] + hf64!("0x1.5555555555555p-3"), + hf64!("0x1.55555555c4d26p-57"), // degree 3, Q_2[3], Q_2[4] + hf64!("0x1.5555555555555p-5"), // degree 4, Q_2[5] + hf64!("0x1.1111111111111p-7"), // degree 5, Q_2[6] + hf64!("0x1.6c16c3fbb4213p-10"), // degree 6, Q_2[7] + hf64!("0x1.a01a023ede0d7p-13"), // degree 7, Q_2[8] +]; + +/// Approximation for the accurate path of `exp(z)` for `z=zh+zl`, with +/// `|z| < 0.000130273 < 2^-12.88` and `|zl| < 2^-42.6` (assuming `x^y` does not overflow +/// or underflow) +#[inline] +fn q_2(zh: f64, zl: f64) -> (f64, f64) { + // Let q[0]..q[7] be the coefficients of degree 0..7 of Q_2. + // The ulp of q[7]*z^7 is at most 2^-155, thus we can compute q[7]*z^7 + // in double precision only. + // The ulp of q[6]*z^6 is at most 2^-139, thus we can compute q[6]*z^6 + // in double precision only. + // The ulp of q[5]*z^5 is at most 2^-124, thus we can compute q[5]*z^5 + // in double precision only. + let z = zh + zl; + let q = Q_2[8].fma(zh, Q_2[7]); + let q = q.fma(z, Q_2[6]); + let q = q.fma(z, Q_2[5]); + + // multiply q by z and add Q_2[3] + Q_2[4] + let (hi, lo) = a_mul(q, z); + let (hi, t) = fast_two_sum(Q_2[3], hi); + let lo = lo + (t + Q_2[4]); + + // multiply hi+lo by zh+zl and add Q_2[2] + let (hi, lo) = d_mul(hi, lo, zh, zl); + let (hi, lo) = fast_sum(Q_2[2], hi, lo); + // multiply hi+lo by zh+zl and add Q_2[1] + let (hi, lo) = d_mul(hi, lo, zh, zl); + let (hi, lo) = fast_sum(Q_2[1], hi, lo); + // multiply hi+lo by zh+zl and add Q_2[0] + let (hi, lo) = d_mul(hi, lo, zh, zl); + fast_sum(Q_2[0], hi, lo) +} + +/// Add `a + b`, assuming `|a| >= |b|` +#[inline] +fn fast_two_sum(a: f64, b: f64) -> (f64, f64) { + let hi = a + b; + let e = hi - a; // exact + let lo = b - e; // exact + (hi, lo) +} + +/// Add `a + (bh + bl)`, assuming `|a| >= |bh|` +#[inline] +fn fast_sum(a: f64, bh: f64, bl: f64) -> (f64, f64) { + let (hi, lo) = fast_two_sum(a, bh); + // |(a+bh)-(hi+lo)| <= 2^-105 |hi| and |lo| < ulp(hi) + let lo = lo + bl; + // |(a+bh+bl)-(hi+lo)| <= 2^-105 |hi| + ulp(lo), + // where |lo| <= ulp(hi) + |bl|. + (hi, lo) +} + +/// Returns `(ah + al) * (bh + bl) - (al * bl)` +#[inline] +fn d_mul(ah: f64, al: f64, bh: f64, bl: f64) -> (f64, f64) { + let (hi, lo) = a_mul(ah, bh); + let lo = ah.fma(bl, lo); + let lo = al.fma(bh, lo); + (hi, lo) +} + +/// Multiply a double with a double double : `a * (bh + bl)` +#[inline] +fn s_mul(a: f64, bh: f64, bl: f64) -> (f64, f64) { + let (hi, lo) = a_mul(a, bh); // exact + let lo = a.fma(bl, lo); + (hi, lo) +} + +/// Multiply exactly `a` and `b`, such that `hi + lo = a * b`. +#[inline] +fn a_mul(a: f64, b: f64) -> (f64, f64) { + let hi = a * b; + let lo = a.fma(b, -hi); + (hi, lo) +} + +/// Return the rounded sum `hi + lo` for `x` from a table sorted by input value. +/// Each entry contains `(input, hi, lo)`. Return `None` if `x` is absent. +#[inline] +fn lookup_exception(mut table: &[(f64, f64, f64)], x: f64) -> Option { + while !table.is_empty() { + // Invariant: if x is an exceptional case, we have + // table[0].0 <= x and x <= table[table.len() - 1].0 + let middle = table.len() / 2; + let (input, hi, lo) = table[middle]; + + if x == input { + return Some(hi + lo); + } + + if x < input { + table = &table[..middle]; + } else { + table = &table[middle + 1..]; + } + } + + None +} + +/// For `0 <= i < 64`, `T1[i] = (h,l)` such that `h+l` is the best double-double approximation +/// of `2^(i/64)`. The approximation error is bounded as follows: `|h + l - 2^(i/64)| < 2^-107`. +#[rustfmt::skip] +static T1: [(f64, f64); 64] = [ + (hf64!("0x1p+0"), hf64!("0x0p+0")), + (hf64!("0x1.02c9a3e778061p+0"), -hf64!("0x1.19083535b085dp-56")), + (hf64!("0x1.059b0d3158574p+0"), hf64!("0x1.d73e2a475b465p-55")), + (hf64!("0x1.0874518759bc8p+0"), hf64!("0x1.186be4bb284ffp-57")), + (hf64!("0x1.0b5586cf9890fp+0"), hf64!("0x1.8a62e4adc610bp-54")), + (hf64!("0x1.0e3ec32d3d1a2p+0"), hf64!("0x1.03a1727c57b53p-59")), + (hf64!("0x1.11301d0125b51p+0"), -hf64!("0x1.6c51039449b3ap-54")), + (hf64!("0x1.1429aaea92dep+0"), -hf64!("0x1.32fbf9af1369ep-54")), + (hf64!("0x1.172b83c7d517bp+0"), -hf64!("0x1.19041b9d78a76p-55")), + (hf64!("0x1.1a35beb6fcb75p+0"), hf64!("0x1.e5b4c7b4968e4p-55")), + (hf64!("0x1.1d4873168b9aap+0"), hf64!("0x1.e016e00a2643cp-54")), + (hf64!("0x1.2063b88628cd6p+0"), hf64!("0x1.dc775814a8495p-55")), + (hf64!("0x1.2387a6e756238p+0"), hf64!("0x1.9b07eb6c70573p-54")), + (hf64!("0x1.26b4565e27cddp+0"), hf64!("0x1.2bd339940e9d9p-55")), + (hf64!("0x1.29e9df51fdee1p+0"), hf64!("0x1.612e8afad1255p-55")), + (hf64!("0x1.2d285a6e4030bp+0"), hf64!("0x1.0024754db41d5p-54")), + (hf64!("0x1.306fe0a31b715p+0"), hf64!("0x1.6f46ad23182e4p-55")), + (hf64!("0x1.33c08b26416ffp+0"), hf64!("0x1.32721843659a6p-54")), + (hf64!("0x1.371a7373aa9cbp+0"), -hf64!("0x1.63aeabf42eae2p-54")), + (hf64!("0x1.3a7db34e59ff7p+0"), -hf64!("0x1.5e436d661f5e3p-56")), + (hf64!("0x1.3dea64c123422p+0"), hf64!("0x1.ada0911f09ebcp-55")), + (hf64!("0x1.4160a21f72e2ap+0"), -hf64!("0x1.ef3691c309278p-58")), + (hf64!("0x1.44e086061892dp+0"), hf64!("0x1.89b7a04ef80dp-59")), + (hf64!("0x1.486a2b5c13cdp+0"), hf64!("0x1.3c1a3b69062fp-56")), + (hf64!("0x1.4bfdad5362a27p+0"), hf64!("0x1.d4397afec42e2p-56")), + (hf64!("0x1.4f9b2769d2ca7p+0"), -hf64!("0x1.4b309d25957e3p-54")), + (hf64!("0x1.5342b569d4f82p+0"), -hf64!("0x1.07abe1db13cadp-55")), + (hf64!("0x1.56f4736b527dap+0"), hf64!("0x1.9bb2c011d93adp-54")), + (hf64!("0x1.5ab07dd485429p+0"), hf64!("0x1.6324c054647adp-54")), + (hf64!("0x1.5e76f15ad2148p+0"), hf64!("0x1.ba6f93080e65ep-54")), + (hf64!("0x1.6247eb03a5585p+0"), -hf64!("0x1.383c17e40b497p-54")), + (hf64!("0x1.6623882552225p+0"), -hf64!("0x1.bb60987591c34p-54")), + (hf64!("0x1.6a09e667f3bcdp+0"), -hf64!("0x1.bdd3413b26456p-54")), + (hf64!("0x1.6dfb23c651a2fp+0"), -hf64!("0x1.bbe3a683c88abp-57")), + (hf64!("0x1.71f75e8ec5f74p+0"), -hf64!("0x1.16e4786887a99p-55")), + (hf64!("0x1.75feb564267c9p+0"), -hf64!("0x1.0245957316dd3p-54")), + (hf64!("0x1.7a11473eb0187p+0"), -hf64!("0x1.41577ee04992fp-55")), + (hf64!("0x1.7e2f336cf4e62p+0"), hf64!("0x1.05d02ba15797ep-56")), + (hf64!("0x1.82589994cce13p+0"), -hf64!("0x1.d4c1dd41532d8p-54")), + (hf64!("0x1.868d99b4492edp+0"), -hf64!("0x1.fc6f89bd4f6bap-54")), + (hf64!("0x1.8ace5422aa0dbp+0"), hf64!("0x1.6e9f156864b27p-54")), + (hf64!("0x1.8f1ae99157736p+0"), hf64!("0x1.5cc13a2e3976cp-55")), + (hf64!("0x1.93737b0cdc5e5p+0"), -hf64!("0x1.75fc781b57ebcp-57")), + (hf64!("0x1.97d829fde4e5p+0"), -hf64!("0x1.d185b7c1b85d1p-54")), + (hf64!("0x1.9c49182a3f09p+0"), hf64!("0x1.c7c46b071f2bep-56")), + (hf64!("0x1.a0c667b5de565p+0"), -hf64!("0x1.359495d1cd533p-54")), + (hf64!("0x1.a5503b23e255dp+0"), -hf64!("0x1.d2f6edb8d41e1p-54")), + (hf64!("0x1.a9e6b5579fdbfp+0"), hf64!("0x1.0fac90ef7fd31p-54")), + (hf64!("0x1.ae89f995ad3adp+0"), hf64!("0x1.7a1cd345dcc81p-54")), + (hf64!("0x1.b33a2b84f15fbp+0"), -hf64!("0x1.2805e3084d708p-57")), + (hf64!("0x1.b7f76f2fb5e47p+0"), -hf64!("0x1.5584f7e54ac3bp-56")), + (hf64!("0x1.bcc1e904bc1d2p+0"), hf64!("0x1.23dd07a2d9e84p-55")), + (hf64!("0x1.c199bdd85529cp+0"), hf64!("0x1.11065895048ddp-55")), + (hf64!("0x1.c67f12e57d14bp+0"), hf64!("0x1.2884dff483cadp-54")), + (hf64!("0x1.cb720dcef9069p+0"), hf64!("0x1.503cbd1e949dbp-56")), + (hf64!("0x1.d072d4a07897cp+0"), -hf64!("0x1.cbc3743797a9cp-54")), + (hf64!("0x1.d5818dcfba487p+0"), hf64!("0x1.2ed02d75b3707p-55")), + (hf64!("0x1.da9e603db3285p+0"), hf64!("0x1.c2300696db532p-54")), + (hf64!("0x1.dfc97337b9b5fp+0"), -hf64!("0x1.1a5cd4f184b5cp-54")), + (hf64!("0x1.e502ee78b3ff6p+0"), hf64!("0x1.39e8980a9cc8fp-55")), + (hf64!("0x1.ea4afa2a490dap+0"), -hf64!("0x1.e9c23179c2893p-54")), + (hf64!("0x1.efa1bee615a27p+0"), hf64!("0x1.dc7f486a4b6bp-54")), + (hf64!("0x1.f50765b6e454p+0"), hf64!("0x1.9d3e12dd8a18bp-54")), + (hf64!("0x1.fa7c1819e90d8p+0"), hf64!("0x1.74853f3a5931ep-55")), +]; + +/// For `0 <= i < 64`, `T2[i] = (h,l)` such that `h+l` is the best double-double approximation +/// of `2^(i/2^12)`. The approximation error is bounded as follows: +/// `|h + l - 2^(i/2^12)| < 2^-107`. +#[rustfmt::skip] +static T2: [(f64, f64); 64] = [ + (hf64!("0x1p+0"), hf64!("0x0p+0")), + (hf64!("0x1.000b175effdc7p+0"), hf64!("0x1.ae8e38c59c72ap-54")), + (hf64!("0x1.00162f3904052p+0"), -hf64!("0x1.7b5d0d58ea8f4p-58")), + (hf64!("0x1.0021478e11ce6p+0"), hf64!("0x1.4115cb6b16a8ep-54")), + (hf64!("0x1.002c605e2e8cfp+0"), -hf64!("0x1.d7c96f201bb2fp-55")), + (hf64!("0x1.003779a95f959p+0"), hf64!("0x1.84711d4c35e9fp-54")), + (hf64!("0x1.0042936faa3d8p+0"), -hf64!("0x1.0484245243777p-55")), + (hf64!("0x1.004dadb113dap+0"), -hf64!("0x1.4b237da2025f9p-54")), + (hf64!("0x1.0058c86da1c0ap+0"), -hf64!("0x1.5e00e62d6b30dp-56")), + (hf64!("0x1.0063e3a559473p+0"), hf64!("0x1.a1d6cedbb9481p-54")), + (hf64!("0x1.006eff583fc3dp+0"), -hf64!("0x1.4acf197a00142p-54")), + (hf64!("0x1.007a1b865a8cap+0"), -hf64!("0x1.eaf2ea42391a5p-57")), + (hf64!("0x1.0085382faef83p+0"), hf64!("0x1.da93f90835f75p-56")), + (hf64!("0x1.00905554425d4p+0"), -hf64!("0x1.6a79084ab093cp-55")), + (hf64!("0x1.009b72f41a12bp+0"), hf64!("0x1.86364f8fbe8f8p-54")), + (hf64!("0x1.00a6910f3b6fdp+0"), -hf64!("0x1.82e8e14e3110ep-55")), + (hf64!("0x1.00b1afa5abcbfp+0"), -hf64!("0x1.4f6b2a7609f71p-55")), + (hf64!("0x1.00bcceb7707ecp+0"), -hf64!("0x1.e1a258ea8f71bp-56")), + (hf64!("0x1.00c7ee448ee02p+0"), hf64!("0x1.4362ca5bc26f1p-56")), + (hf64!("0x1.00d30e4d0c483p+0"), hf64!("0x1.095a56c919d02p-54")), + (hf64!("0x1.00de2ed0ee0f5p+0"), -hf64!("0x1.406ac4e81a645p-57")), + (hf64!("0x1.00e94fd0398ep+0"), hf64!("0x1.b5a6902767e09p-54")), + (hf64!("0x1.00f4714af41d3p+0"), -hf64!("0x1.91b2060859321p-54")), + (hf64!("0x1.00ff93412315cp+0"), hf64!("0x1.427068ab22306p-55")), + (hf64!("0x1.010ab5b2cbd11p+0"), hf64!("0x1.c1d0660524e08p-54")), + (hf64!("0x1.0115d89ff3a8bp+0"), -hf64!("0x1.e7bdfb3204be8p-54")), + (hf64!("0x1.0120fc089ff63p+0"), hf64!("0x1.843aa8b9cbbc6p-55")), + (hf64!("0x1.012c1fecd613bp+0"), -hf64!("0x1.34104ee7edae9p-56")), + (hf64!("0x1.0137444c9b5b5p+0"), -hf64!("0x1.2b6aeb6176892p-56")), + (hf64!("0x1.01426927f5278p+0"), hf64!("0x1.a8cd33b8a1bb3p-56")), + (hf64!("0x1.014d8e7ee8d2fp+0"), hf64!("0x1.2edc08e5da99ap-56")), + (hf64!("0x1.0158b4517bb88p+0"), hf64!("0x1.57ba2dc7e0c73p-55")), + (hf64!("0x1.0163da9fb3335p+0"), hf64!("0x1.b61299ab8cdb7p-54")), + (hf64!("0x1.016f0169949edp+0"), -hf64!("0x1.90565902c5f44p-54")), + (hf64!("0x1.017a28af25567p+0"), hf64!("0x1.70fc41c5c2d53p-55")), + (hf64!("0x1.018550706ab62p+0"), hf64!("0x1.4b9a6e145d76cp-54")), + (hf64!("0x1.019078ad6a19fp+0"), -hf64!("0x1.008eff5142bf9p-56")), + (hf64!("0x1.019ba16628de2p+0"), -hf64!("0x1.77669f033c7dep-54")), + (hf64!("0x1.01a6ca9aac5f3p+0"), -hf64!("0x1.09bb78eeead0ap-54")), + (hf64!("0x1.01b1f44af9f9ep+0"), hf64!("0x1.371231477ece5p-54")), + (hf64!("0x1.01bd1e77170b4p+0"), hf64!("0x1.5e7626621eb5bp-56")), + (hf64!("0x1.01c8491f08f08p+0"), -hf64!("0x1.bc72b100828a5p-54")), + (hf64!("0x1.01d37442d507p+0"), -hf64!("0x1.ce39cbbab8bbep-57")), + (hf64!("0x1.01de9fe280ac8p+0"), hf64!("0x1.16996709da2e2p-55")), + (hf64!("0x1.01e9cbfe113efp+0"), -hf64!("0x1.c11f5239bf535p-55")), + (hf64!("0x1.01f4f8958c1c6p+0"), hf64!("0x1.e1d4eb5edc6b3p-55")), + (hf64!("0x1.020025a8f6a35p+0"), -hf64!("0x1.afb99946ee3fp-54")), + (hf64!("0x1.020b533856324p+0"), -hf64!("0x1.8f06d8a148a32p-54")), + (hf64!("0x1.02168143b0281p+0"), -hf64!("0x1.2bf310fc54eb6p-55")), + (hf64!("0x1.0221afcb09e3ep+0"), -hf64!("0x1.c95a035eb4175p-54")), + (hf64!("0x1.022cdece68c4fp+0"), -hf64!("0x1.491793e46834dp-54")), + (hf64!("0x1.02380e4dd22adp+0"), -hf64!("0x1.3e8d0d9c49091p-56")), + (hf64!("0x1.02433e494b755p+0"), -hf64!("0x1.314aa16278aa3p-54")), + (hf64!("0x1.024e6ec0da046p+0"), hf64!("0x1.48daf888e9651p-55")), + (hf64!("0x1.02599fb483385p+0"), hf64!("0x1.56dc8046821f4p-55")), + (hf64!("0x1.0264d1244c719p+0"), hf64!("0x1.45b42356b9d47p-54")), + (hf64!("0x1.027003103b10ep+0"), -hf64!("0x1.082ef51b61d7ep-56")), + (hf64!("0x1.027b357854772p+0"), hf64!("0x1.2106ed0920a34p-56")), + (hf64!("0x1.0286685c9e059p+0"), -hf64!("0x1.fd4cf26ea5d0fp-54")), + (hf64!("0x1.02919bbd1d1d8p+0"), -hf64!("0x1.09f8775e78084p-54")), + (hf64!("0x1.029ccf99d720ap+0"), hf64!("0x1.64cbba902ca27p-58")), + (hf64!("0x1.02a803f2d170dp+0"), hf64!("0x1.4383ef231d207p-54")), + (hf64!("0x1.02b338c811703p+0"), hf64!("0x1.4a47a505b3a47p-54")), + (hf64!("0x1.02be6e199c811p+0"), hf64!("0x1.e47120223467fp-54")), +]; + +#[cfg(test)] +mod tests { + use super::exp2m1; + use crate::support::{CastFrom, CastInto, Float}; + + #[test] + fn known_results() { + for (x, expected) in [ + (f64::ZERO, f64::ZERO), + (f64::NEG_ZERO, f64::NEG_ZERO), + (f64::ONE, f64::ONE), + (f64::NEG_ONE, -0.5), + (f64::from_bits(1), f64::from_bits(1)), + (-f64::from_bits(1), -f64::from_bits(1)), + (f64::from_bits(2), f64::from_bits(1)), + (-f64::from_bits(2), -f64::from_bits(1)), + (54.0, hf64!("0x1p+54")), + (-54.0, f64::NEG_ONE), + (1023.0, hf64!("0x1p+1023")), + (1024.0, f64::INFINITY), + (f64::MAX, f64::INFINITY), + (f64::MIN, f64::NEG_ONE), + (f64::INFINITY, f64::INFINITY), + (f64::NEG_INFINITY, f64::NEG_ONE), + ] { + if !cfg!(x86_no_sse2) { + let result = exp2m1(x); + assert_biteq!(result, expected, "x = {x:?}"); + } + } + + for x in [f64::NAN, f64::NEG_NAN, f64::SNAN, f64::NEG_SNAN] { + // NaN operations can change the sign or payload, including canonicalization on RISC-V. + assert!(exp2m1(x).is_nan(), "x = {x:?}"); + } + } + + #[test] + fn exact_integer_results() { + for i in 1..=53 { + let x = f64::cast_from(i); + let positive = ((1u64 << i) - 1).cast(); + let negative = f64::from_bits((1023 - i) << 52) - 1.0; + + assert_biteq!(exp2m1(x), positive, "x = {x:?}"); + assert_biteq!(exp2m1(-x), negative, "x = {:?}", -x); + } + } +} diff --git a/library/compiler-builtins/libm/src/math/exp2m1f.rs b/library/compiler-builtins/libm/src/math/exp2m1f.rs new file mode 100644 index 0000000000000..2c941f909b572 --- /dev/null +++ b/library/compiler-builtins/libm/src/math/exp2m1f.rs @@ -0,0 +1,257 @@ +/* SPDX-License-Identifier: MIT */ +/* origin: core-math/src/binary32/exp2m1/exp2m1f.c + * Copyright (c) 2022-2025 Alexei Sibidanov. + * Ported to Rust in 2026, malezjaa + * Approximate CORE-MATH commit: 8ea8ea35c518d06b72de4cf1d528e7a2d16f104c + */ + +use super::support::{CastFrom, CastInto, Float, Int, cold_path}; + +/// Base-2 exponential minus one (f32). +/// +/// Calculate `2^x - 1`, that is, 2 raised to the power `x` minus `1`. +#[cfg_attr(assert_no_panic, no_panic::no_panic)] +pub fn exp2m1f(x: f32) -> f32 { + cr_exp2m1f(x) +} + +fn cr_exp2m1f(x: f32) -> f32 { + const Q: [(f32, f32); 3] = [ + (hf32!("0x1.fffffep127"), hf32!("0x1.fffffep127")), + (hf32!("0x1.fffffep127"), hf32!("0x1p+103")), + (-1.0, hf32!("0x1p-26")), + ]; + + let z = f64::from(x); + let ux = x.to_bits(); + let ax = ux & u32::MAX >> 1; + + // x <= -25 + if ux >= 0xc1c8_0000 { + cold_path(); + if ax > (0xff << 23) { + return x + x; // NaN + } + // avoid spurious inexact exception for -Inf + if ux == 0xff800000 { + return Q[2].0; + } else { + return Q[2].0 + Q[2].1; + } + } + + // x >= 128, result is infinite (with default rounding) + if ax >= 0x4300_0000 { + cold_path(); + if ax > (0xff << 23) { + return x + x; // nan + } + // for x=128 and rounding downward or to zero, there is no overflow + let special = usize::from(x == 128.0 && (Q[1].0 + Q[1].1 == Q[1].0)); + // avoid spurious inexact exception for +Inf + if ux == 0x7f800000 { + return x; + } else { + return Q[special].0 + Q[special].1; + } + } + + if ax < 0x3df95f1f { + // |x| < 8.44e-2/log(2) + cold_path(); + let z2 = z * z; + let mut r: f64; + + if ax < 0x3d67a4cc { + // |x| < 3.92e-2/log(2) + cold_path(); + if ax < 0x3caa2fee { + // |x| < 1.44e-2/log(2) + cold_path(); + if ax < 0x3bac1405 { + // |x| < 3.64e-3/log(2) + cold_path(); + if ax < 0x3a358876 { + // |x| < 4.8e-4/log(2) + cold_path(); + if ax < 0x37d32ef6 { + // |x| < 1.745e-5/log(2) + cold_path(); + if ax < 0x331fdd82 { + // |x| < 2.58e-8/log(2) + cold_path(); + if ax < 0x2538aa3b { + // |x| < 0x1.715476p-53 + cold_path(); + r = hf64!("0x1.62e42fefa39efp-1"); + } else { + r = hf64!("0x1.62e42fefa39fp-1") + + z * hf64!("0x1.ebfbdff82c58fp-3"); + } + } else { + if ux == 0xb3d8_5005 { + cold_path(); + return (-hf64!("0x1.2bdf76p-24") - hf64!("0x1.8p-77")) + .cast_lossy(); + } + + if ux == 0x3338_428d { + cold_path(); + return (hf64!("0x1.fee08ap-26") + hf64!("0x1p-80")) + .cast_lossy(); + } + + const C: [f64; 3] = [ + hf64!("0x1.62e42fefa39efp-1"), + hf64!("0x1.ebfbdff8548fdp-3"), + hf64!("0x1.c6b08d704a06dp-5"), + ]; + + r = C[0] + z * (C[1] + z * C[2]); + } + } else { + if ux == 0x388bca4f { + cold_path(); + return (hf64!("0x1.839702p-15") - hf64!("0x1.8p-68")).cast_lossy(); + } + + const C: [f64; 4] = [ + hf64!("0x1.62e42fefa39efp-1"), + hf64!("0x1.ebfbdff82c58fp-3"), + hf64!("0x1.c6b08dc82b347p-5"), + hf64!("0x1.3b2ab6fbad172p-7"), + ]; + r = (C[0] + z * C[1]) + z2 * (C[2] + z * C[3]); + } + } else { + const C: [f64; 5] = [ + hf64!("0x1.62e42fefa39efp-1"), + hf64!("0x1.ebfbdff82c068p-3"), + hf64!("0x1.c6b08d704a6dcp-5"), + hf64!("0x1.3b2ac262c3eedp-7"), + hf64!("0x1.5d87fe7af779ap-10"), + ]; + r = (C[0] + z * C[1]) + z2 * (C[2] + z * (C[3] + z * C[4])); + } + } else { + const C: [f64; 6] = [ + hf64!("0x1.62e42fefa39fp-1"), + hf64!("0x1.ebfbdff82c58dp-3"), + hf64!("0x1.c6b08d7011d13p-5"), + hf64!("0x1.3b2ab6fbd267dp-7"), + hf64!("0x1.5d88a81cea49ep-10"), + hf64!("0x1.430912ea9b963p-13"), + ]; + r = (C[0] + z * C[1]) + z2 * ((C[2] + z * C[3]) + z2 * (C[4] + z * C[5])); + } + } else { + const C: [f64; 7] = [ + hf64!("0x1.62e42fefa39efp-1"), + hf64!("0x1.ebfbdff82c639p-3"), + hf64!("0x1.c6b08d7049f1cp-5"), + hf64!("0x1.3b2ab6f5243bdp-7"), + hf64!("0x1.5d87fe80a9e6cp-10"), + hf64!("0x1.430d0b9257fa8p-13"), + hf64!("0x1.ffcbfc4cf0952p-17"), + ]; + r = (C[0] + z * C[1]) + + z2 * ((C[2] + z * C[3]) + z2 * (C[4] + z * (C[5] + z * C[6]))); + } + } else { + const C: [f64; 8] = [ + hf64!("0x1.62e42fefa39efp-1"), + hf64!("0x1.ebfbdff82c591p-3"), + hf64!("0x1.c6b08d704cf6bp-5"), + hf64!("0x1.3b2ab6fba00cep-7"), + hf64!("0x1.5d87fdfdaadb4p-10"), + hf64!("0x1.4309137333066p-13"), + hf64!("0x1.ffe5e90daf7ddp-17"), + hf64!("0x1.62c0220eed731p-20"), + ]; + + r = ((C[0] + z * C[1]) + z2 * (C[2] + z * C[3])) + + (z2 * z2) * ((C[4] + z * C[5]) + z2 * (C[6] + z * C[7])); + } + + r *= z; + r.cast_lossy() + } else { + const C: [f64; 6] = [ + hf64!("0x1.62e42fefa398bp-5"), + hf64!("0x1.ebfbdff84555ap-11"), + hf64!("0x1.c6b08d4ad86d3p-17"), + hf64!("0x1.3b2ad1b1716a2p-23"), + hf64!("0x1.5d7472718ce9dp-30"), + hf64!("0x1.4a1d7f457ac56p-37"), + ]; + + const TB: [f64; 16] = [ + hf64!("0x1p+0"), + hf64!("0x1.0b5586cf9890fp+0"), + hf64!("0x1.172b83c7d517bp+0"), + hf64!("0x1.2387a6e756238p+0"), + hf64!("0x1.306fe0a31b715p+0"), + hf64!("0x1.3dea64c123422p+0"), + hf64!("0x1.4bfdad5362a27p+0"), + hf64!("0x1.5ab07dd485429p+0"), + hf64!("0x1.6a09e667f3bcdp+0"), + hf64!("0x1.7a11473eb0187p+0"), + hf64!("0x1.8ace5422aa0dap+0"), + hf64!("0x1.9c49182a3f09p+0"), + hf64!("0x1.ae89f995ad3adp+0"), + hf64!("0x1.c199bdd85529cp+0"), + hf64!("0x1.d5818dcfba487p+0"), + hf64!("0x1.ea4afa2a490dap+0"), + ]; + + let a = 16.0 * z; + let ia = Float::floor(a); + let h = a - ia; + let h2 = h * h; + + let i: i64 = ia.cast(); + let j = i & 0xf; + let mut e = i - j; + e >>= 4; + + let mut s = TB[usize::cast_from(j)]; + let su = f64::from_bits((e + 0x3ff).unsigned() << 52); + s *= su; + + let mut c0 = C[0] + h * C[1]; + let c2 = C[2] + h * C[3]; + let c4 = C[4] + h * C[5]; + + c0 += h2 * (c2 + h2 * c4); + let w = s * h; + ((s - 1.0) + w * c0).cast_lossy() + } +} + +#[cfg(test)] +mod tests { + use super::exp2m1f; + use crate::support::Float; + + #[test] + fn known_results() { + for (x, expected) in [ + (f32::ZERO, f32::ZERO), + (f32::NEG_ZERO, f32::NEG_ZERO), + (f32::ONE, f32::ONE), + (f32::NEG_ONE, -0.5), + (128.0, f32::INFINITY), + (-25.0, f32::NEG_ONE), + (f32::INFINITY, f32::INFINITY), + (f32::NEG_INFINITY, f32::NEG_ONE), + ] { + let result = exp2m1f(x); + assert_biteq!(result, expected, "x = {x:?}"); + } + + for x in [f32::NAN, f32::NEG_NAN, f32::SNAN, f32::NEG_SNAN] { + // NaN operations can change the sign or payload, including canonicalization on RISC-V. + assert!(exp2m1f(x).is_nan(), "x = {x:?}"); + } + } +} diff --git a/library/compiler-builtins/libm/src/math/fdim.rs b/library/compiler-builtins/libm/src/math/fdim.rs index b61630847f808..82aa7fb081e63 100644 --- a/library/compiler-builtins/libm/src/math/fdim.rs +++ b/library/compiler-builtins/libm/src/math/fdim.rs @@ -56,6 +56,7 @@ pub fn fdimf128(x: f128, y: f128) -> f128 { mod tests { use crate::support::{Float, Hex}; + // FIXME: bitwise tests including NaN are skipped because of failure for f128 macro_rules! cases { ($f:ty) => { [ @@ -66,7 +67,7 @@ mod tests { (<$f>::NEG_INFINITY, 0.0, 0.0), (<$f>::NEG_INFINITY, 1.0, 0.0), (<$f>::NEG_INFINITY, <$f>::INFINITY, 0.0), - (<$f>::NEG_INFINITY, <$f>::NAN, <$f>::NAN), + // (<$f>::NEG_INFINITY, <$f>::NAN, <$f>::NAN), // x = -1.0 (-1.0, <$f>::NEG_INFINITY, <$f>::INFINITY), (-1.0, -1.0, 0.0), @@ -74,7 +75,7 @@ mod tests { (-1.0, 0.0, 0.0), (-1.0, 1.0, 0.0), (-1.0, <$f>::INFINITY, 0.0), - (-1.0, <$f>::NAN, <$f>::NAN), + // (-1.0, <$f>::NAN, <$f>::NAN), // x = -0.0 (-0.0, <$f>::NEG_INFINITY, <$f>::INFINITY), (-0.0, -1.0, 1.0), @@ -82,7 +83,7 @@ mod tests { (-0.0, 0.0, 0.0), (-0.0, 1.0, 0.0), (-0.0, <$f>::INFINITY, 0.0), - (-0.0, <$f>::NAN, <$f>::NAN), + // (-0.0, <$f>::NAN, <$f>::NAN), // x = 0.0 (0.0, <$f>::NEG_INFINITY, <$f>::INFINITY), (0.0, -1.0, 1.0), @@ -90,7 +91,7 @@ mod tests { (0.0, 0.0, 0.0), (0.0, 1.0, 0.0), (0.0, <$f>::INFINITY, 0.0), - (0.0, <$f>::NAN, <$f>::NAN), + // (0.0, <$f>::NAN, <$f>::NAN), // x = 1.0 (1.0, <$f>::NEG_INFINITY, <$f>::INFINITY), (1.0, -1.0, 2.0), @@ -98,7 +99,7 @@ mod tests { (1.0, 0.0, 1.0), (1.0, 1.0, 0.0), (1.0, <$f>::INFINITY, 0.0), - (1.0, <$f>::NAN, <$f>::NAN), + // (1.0, <$f>::NAN, <$f>::NAN), // x = inf (<$f>::INFINITY, <$f>::NEG_INFINITY, <$f>::INFINITY), (<$f>::INFINITY, -1.0, <$f>::INFINITY), @@ -106,7 +107,7 @@ mod tests { (<$f>::INFINITY, 0.0, <$f>::INFINITY), (<$f>::INFINITY, 1.0, <$f>::INFINITY), (<$f>::INFINITY, <$f>::INFINITY, 0.0), - (<$f>::INFINITY, <$f>::NAN, <$f>::NAN), + // (<$f>::INFINITY, <$f>::NAN, <$f>::NAN), // x = nan (<$f>::NAN, <$f>::NEG_INFINITY, <$f>::NAN), (<$f>::NAN, -1.0, <$f>::NAN), diff --git a/library/compiler-builtins/libm/src/math/mod.rs b/library/compiler-builtins/libm/src/math/mod.rs index 51a3417c8b151..a97c1128f7299 100644 --- a/library/compiler-builtins/libm/src/math/mod.rs +++ b/library/compiler-builtins/libm/src/math/mod.rs @@ -141,6 +141,8 @@ mod exp10; mod exp10f; mod exp2; mod exp2f; +mod exp2m1; +mod exp2m1f; mod expf; mod expm1; mod expm1f; @@ -232,6 +234,8 @@ pub use self::erff::{erfcf, erff}; pub use self::exp::exp; pub use self::exp2::exp2; pub use self::exp2f::exp2f; +pub use self::exp2m1::exp2m1; +pub use self::exp2m1f::exp2m1f; pub use self::exp10::exp10; pub use self::exp10f::exp10f; pub use self::expf::expf; diff --git a/library/compiler-builtins/libm/src/math/support/float_traits.rs b/library/compiler-builtins/libm/src/math/support/float_traits.rs index 43bad1aeb50d0..4da99101b9c31 100644 --- a/library/compiler-builtins/libm/src/math/support/float_traits.rs +++ b/library/compiler-builtins/libm/src/math/support/float_traits.rs @@ -111,6 +111,9 @@ pub trait Float: /// A mask for the top bit of the significand, useful for NaN ops. const SIG_TOP_BIT: Self::Int; + /// The minimum subnormal number. + const TINY_BITS: Self::Int = Self::Int::ONE; + /// Returns `self` transmuted to `Self::Int` fn to_bits(self) -> Self::Int; @@ -220,6 +223,12 @@ pub trait Float: /// Fused multiply add, rounding once. fn fma(self, y: Self, z: Self) -> Self; + /// Round down to the nearest integer. + fn floor(self) -> Self; + + /// Round to the nearest integer, breaking ties toward even. + fn roundeven(self) -> Self; + /// Returns (normalized exponent, normalized significand) fn normalize(significand: Self::Int) -> (i32, Self::Int); @@ -241,6 +250,50 @@ pub trait Float: // ). self * Self::ONE } + + /// Increment by one ULP, saturating at infinity. + fn next_up(self) -> Self { + // FIXME: Replace with `next_up` from `core` once `libm` MSRV reaches 1.86. + let bits = self.to_bits(); + if self.is_nan() || bits == Self::INFINITY.to_bits() { + return self; + } + + let abs = self.abs().to_bits(); + let next_bits = if abs == Self::Int::ZERO { + // Next up from 0 is the smallest subnormal + Self::TINY_BITS + } else if bits == abs { + // Positive: counting up is more positive + bits + Self::Int::ONE + } else { + // Negative: counting down is more positive + bits - Self::Int::ONE + }; + Self::from_bits(next_bits) + } + + /// Decrement by one ULP, saturating at negative infinity. + fn next_down(self) -> Self { + // FIXME: Replace with `next_down` from `core` once `libm` MSRV reaches 1.86. + let bits = self.to_bits(); + if self.is_nan() || bits == Self::NEG_INFINITY.to_bits() { + return self; + } + + let abs = self.abs().to_bits(); + let next_bits = if abs == Self::Int::ZERO { + // Next up from 0 is the smallest negative subnormal + Self::TINY_BITS | Self::SIGN_MASK + } else if bits == abs { + // Positive: counting down is more negative + bits - Self::Int::ONE + } else { + // Negative: counting up is more negative + bits + Self::Int::ONE + }; + Self::from_bits(next_bits) + } } /// Access the associated `Int` type from a float (helper to avoid ambiguous associated types). @@ -256,7 +309,11 @@ macro_rules! float_impl { $from_bits:path, $to_bits:path, $fma_fn:ident, - $fma_intrinsic:ident + $fma_intrinsic:ident, + $floor_fn:ident, + $floor_intrinsic:ident, + $roundeven_fn:ident, + $roundeven_intrinsic:ident ) => { impl Float for $ty { type Int = $ity; @@ -386,6 +443,27 @@ macro_rules! float_impl { } } } + #[inline] + fn floor(self) -> Self { + cfg_select_nofmt! { + intrinsics_enabled => { + core::intrinsics::$floor_intrinsic(self) + } + _ => { + super::super::$floor_fn(self) + } + } + } + fn roundeven(self) -> Self { + cfg_select_nofmt! { + intrinsics_enabled => { + core::intrinsics::$roundeven_intrinsic(self) + } + _ => { + super::super::$roundeven_fn(self) + } + } + } fn normalize(significand: Self::Int) -> (i32, Self::Int) { let shift = significand.leading_zeros().wrapping_sub(Self::EXP_BITS); (1i32.wrapping_sub(shift as i32), significand << shift) @@ -404,7 +482,11 @@ float_impl!( f16::from_bits, f16::to_bits, fmaf16, - fmaf16 + fmaf16, + floorf16, + floorf16, + roundevenf16, + round_ties_even_f16 ); float_impl!( f32, @@ -415,7 +497,11 @@ float_impl!( f32_from_bits, f32_to_bits, fmaf, - fmaf32 + fmaf32, + floorf, + floorf32, + roundevenf, + round_ties_even_f32 ); float_impl!( f64, @@ -426,7 +512,11 @@ float_impl!( f64_from_bits, f64_to_bits, fma, - fmaf64 + fmaf64, + floor, + floorf64, + roundeven, + round_ties_even_f64 ); #[cfg(f128_enabled)] float_impl!( @@ -438,7 +528,11 @@ float_impl!( f128::from_bits, f128::to_bits, fmaf128, - fmaf128 + fmaf128, + floorf128, + floorf128, + roundevenf128, + round_ties_even_f128 ); /* FIXME(msrv): vendor some things that are not const stable at our MSRV */ diff --git a/library/compiler-builtins/libm/src/math/support/int_traits.rs b/library/compiler-builtins/libm/src/math/support/int_traits.rs index 48c0e0438bc1d..922586e46ecbd 100644 --- a/library/compiler-builtins/libm/src/math/support/int_traits.rs +++ b/library/compiler-builtins/libm/src/math/support/int_traits.rs @@ -447,6 +447,7 @@ macro_rules! cast_into { }; ($ty:ty; $($into:ty),*) => {$( impl CastInto<$into> for $ty { + #[inline] fn cast(self) -> $into { // All we can really do to enforce casting rules is check the rules when in // debug mode. @@ -455,6 +456,7 @@ macro_rules! cast_into { self as $into } + #[inline] fn cast_lossy(self) -> $into { self as $into } @@ -474,12 +476,14 @@ macro_rules! cast_into_float { }; ($ty:ty; $($into:ty),*) => {$( impl CastInto<$into> for $ty { + #[inline] fn cast(self) -> $into { #[cfg(not(feature = "compiler-builtins"))] debug_assert_eq!(self as $into as $ty, self, "inexact float cast"); self as $into } + #[inline] fn cast_lossy(self) -> $into { self as $into } @@ -505,3 +509,5 @@ cast_into_float!(i16); cast_into_float!(i32); cast_into_float!(i64); cast_into_float!(i128); +cast_into_float!(u64; f64); +cast_into_float!(f64; f32, i32, i64); diff --git a/library/compiler-builtins/rust-version b/library/compiler-builtins/rust-version index 18fea436747c7..028e73378b28b 100644 --- a/library/compiler-builtins/rust-version +++ b/library/compiler-builtins/rust-version @@ -1 +1 @@ -32d94cc9be3f6e6c3fa1deaea9e0ab93c4980dba +5d8937173e7c40406c167f4e94737da8925333ad diff --git a/library/core/Cargo.toml b/library/core/Cargo.toml index b61474ba7f1d6..0d22ec821474c 100644 --- a/library/core/Cargo.toml +++ b/library/core/Cargo.toml @@ -58,7 +58,7 @@ include = [ "src/flux_info.rs", "src/ascii*.rs", "src/num/mod.rs", "src/pat.rs", - "src/bstr/*.rs", + "src/byte_str/*.rs", "src/hash/*.rs", "src/time.rs", ] diff --git a/library/core/src/arch.rs b/library/core/src/arch.rs index 04fc16f50085b..3d5a53366bd38 100644 --- a/library/core/src/arch.rs +++ b/library/core/src/arch.rs @@ -51,7 +51,8 @@ pub macro global_asm("assembly template", $(operands,)* $(options($(option),*))? /* compiler built-in */ } -/// Compiles to a target-specific software breakpoint instruction or equivalent. +/// Compiles to a target-specific software breakpoint instruction if possible, +/// falling back to an alternative trap or an abort. /// /// This will typically abort the program. It may result in a core dump, and/or the system logging /// debug information. Additional target-specific capabilities may be possible depending on @@ -68,13 +69,20 @@ pub macro global_asm("assembly template", $(operands,)* $(options($(option),*))? /// The precise behavior is not guaranteed, it depends on the architecture and operating system. /// Not all architectures guarantee that a breakpoint instruction interrupts execution in the absence /// of a debugger, and not all operating systems and execution environments guarantee that such an -/// interrupt aborts the current process. +/// interrupt aborts the current process. On operating systems where the behavior of the interrupt +/// can be overwritten by the application (e.g. via a SIGTRAP signal handler), the application +/// *should* make sure to still abort the process. However, code not fully in control of that +/// application state (e.g. signal handlers) cannot always rely on that for soundness. +/// If you do rely on this, such that your code's soundness requires applications to not continue +/// from your breakpoints, please document that for your users. /// /// The precise instruction is guaranteed only on the following targets: /// - On x86 targets, this produces an `int3` instruction. /// - On aarch64 targets, this produces a `brk #0xf000` instruction. -// When adding more items above, also add cases to the test in `tests/assembly-llvm/breakpoint.rs`. +// When adding more items above, also add cases to the test in `tests/assembly-llvm/intrinsics.rs`. // When stabilizing this, update the comment on `core::intrinsics::breakpoint`. +// NOTE: there is an alternative in `process::resumable_breakpoint`. We may or may not +// actually want to stabilize both of them. #[unstable(feature = "breakpoint", issue = "133724")] #[inline(always)] pub fn breakpoint() { diff --git a/library/core/src/array/mod.rs b/library/core/src/array/mod.rs index fcd635a03c652..205c2815be1b4 100644 --- a/library/core/src/array/mod.rs +++ b/library/core/src/array/mod.rs @@ -430,20 +430,12 @@ const impl PartialOrd for [T; N] { } #[inline] - fn __chaining_lt(&self, other: &[T; N]) -> ControlFlow { - <[T] as PartialOrd>::__chaining_lt(self, other) + fn __chaining_less(&self, other: &[T; N]) -> ControlFlow { + <[T] as PartialOrd>::__chaining_less(self, other) } #[inline] - fn __chaining_le(&self, other: &[T; N]) -> ControlFlow { - <[T] as PartialOrd>::__chaining_le(self, other) - } - #[inline] - fn __chaining_ge(&self, other: &[T; N]) -> ControlFlow { - <[T] as PartialOrd>::__chaining_ge(self, other) - } - #[inline] - fn __chaining_gt(&self, other: &[T; N]) -> ControlFlow { - <[T] as PartialOrd>::__chaining_gt(self, other) + fn __chaining_greater(&self, other: &[T; N]) -> ControlFlow { + <[T] as PartialOrd>::__chaining_greater(self, other) } } diff --git a/library/core/src/attribute_docs.rs b/library/core/src/attribute_docs.rs index b84d6c5db977c..dd44057ff3181 100644 --- a/library/core/src/attribute_docs.rs +++ b/library/core/src/attribute_docs.rs @@ -886,3 +886,98 @@ const _: () = (); /// [`cfg`]: ./attribute.cfg.html /// [the `cfg_attr` attribute]: ../reference/conditional-compilation.html#the-cfg_attr-attribute const _: () = (); + +#[doc(attribute = "windows_subsystem")] +// +/// The `windows_subsystem` attribute is used to specify the subsystem for a Windows application. +/// +/// Example: +/// +/// ```rust +/// #![windows_subsystem = "windows"] +/// +/// fn main() {} +/// ``` +/// +/// Accepted values are "console" (default) and "windows". +/// +/// The `windows_subsystem` attribute may only be applied to the crate root. +/// +/// Only the first use of `windows_subsystem` has effect. +/// +/// The `windows_subsystem` attribute is ignored on non-Windows targets and non-bin crate types. +/// If a "console" process is run from an existing console then it will be attached to said console; +/// otherwise a new one will be created. +/// The "windows" subsystem will run detached from any existing "console". +/// +/// [Reference](../reference/runtime.html#the-windows_subsystem-attribute) +const _: () = (); + +#[doc(attribute = "recursion_limit")] +// +/// Sets the maximum depth for recursive compile-time operations. +/// +/// The compiler limits how deeply it recurses while performing operations that could +/// otherwise expand forever, such as macro expansion and auto-dereference. The +/// `recursion_limit` attribute raises that limit. It applies to the whole crate and is +/// written at the crate root: +/// +/// ``` +/// #![recursion_limit = "256"] +/// +/// // Macros may now expand up to 256 levels deep. +/// ``` +/// +/// The value is a string containing the new limit. The default is 128. +/// +/// This limit affects only compile-time recursion. It does not change how deeply +/// recursive code can run at runtime. +/// +/// When a compile-time operation recurses deeper than the limit, compilation fails. For +/// example, this macro expands itself forever, so it always hits the limit: +/// +/// ```compile_fail +/// macro_rules! a { +/// () => { a!(); }; +/// } +/// +/// a!(); +/// ``` +/// +/// This produces an error whose help message suggests how to raise the limit: +/// +/// ```text +/// error: recursion limit reached while expanding `a!` +/// --> src/main.rs:2:13 +/// | +/// 2 | () => { a!(); }; +/// | ^^^^ +/// ... +/// 5 | a!(); +/// | ---- in this macro invocation +/// | +/// = help: consider increasing the recursion limit by adding a `#![recursion_limit = "256"]` +/// attribute to your crate +/// ``` +/// +/// Auto-dereference is limited in the same way: +/// +/// ```compile_fail +/// #![recursion_limit = "1"] +/// +/// // Error: two recursive steps are needed to auto-dereference `&&&1` to `&u8`. +/// let f = |_: &u8| {}; +/// f(&&&1); +/// ``` +/// +/// Reaching the limit is most common in code that nests many macros, such as code expanded +/// by `derive` macros or deeply nested `async` code. Raising the limit is safe, but very +/// large values can make the compiler use more time and memory, and deeply nested code can +/// still overflow the compiler's own stack. Increase the limit in measured steps rather +/// than jumping to a very large number. If the limit is reached without an obvious cause, +/// the code may be more deeply recursive than intended. +/// +/// For more information, see the Reference on [the `recursion_limit` attribute]. +/// +/// [the `recursion_limit` attribute]: ../reference/attributes/limits.html#the-recursion_limit-attribute +const _: () = (); diff --git a/library/core/src/bstr/mod.rs b/library/core/src/byte_str/mod.rs similarity index 85% rename from library/core/src/bstr/mod.rs rename to library/core/src/byte_str/mod.rs index 0530ddc292a99..9ce00b813b6d8 100644 --- a/library/core/src/bstr/mod.rs +++ b/library/core/src/byte_str/mod.rs @@ -2,7 +2,7 @@ mod traits; -#[unstable(feature = "bstr_internals", issue = "none")] +#[unstable(feature = "byte_str_internals", issue = "none")] pub use traits::{impl_partial_eq, impl_partial_eq_n, impl_partial_eq_ord}; use crate::borrow::{Borrow, BorrowMut}; @@ -17,7 +17,7 @@ use crate::ops::{Deref, DerefMut, DerefPure}; /// need to round-trip whatever data the user provides. /// /// For an owned, growable byte string buffer, use -/// [`ByteString`](../../std/bstr/struct.ByteString.html). +/// [`ByteString`](../../std/byte_str/struct.ByteString.html). /// /// `ByteStr` implements `Deref` to `[u8]`, so all methods available on `[u8]` are available on /// `ByteStr`. @@ -37,7 +37,7 @@ use crate::ops::{Deref, DerefMut, DerefPure}; /// /// The `Display` implementation behaves as if the `ByteStr` were first lossily converted to a /// `str`, with invalid UTF-8 presented as the Unicode replacement character (�). -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[rustc_has_incoherent_inherent_impls] #[repr(transparent)] #[doc(alias = "BStr")] @@ -53,8 +53,8 @@ impl ByteStr { /// You can create a `ByteStr` from a byte array, a byte slice or a string slice: /// /// ``` - /// # #![feature(bstr)] - /// # use std::bstr::ByteStr; + /// # #![feature(byte_str)] + /// # use std::byte_str::ByteStr; /// let a = ByteStr::new(b"abc"); /// let b = ByteStr::new(&b"abc"[..]); /// let c = ByteStr::new("abc"); @@ -63,7 +63,7 @@ impl ByteStr { /// assert_eq!(a, c); /// ``` #[inline] - #[unstable(feature = "bstr", issue = "134915")] + #[unstable(feature = "byte_str", issue = "134915")] #[rustc_const_unstable(feature = "const_convert", issue = "143773")] pub const fn new>(bytes: &B) -> &Self { ByteStr::from_bytes(bytes.as_ref()) @@ -76,7 +76,7 @@ impl ByteStr { /// for example `Box` or `Arc`. #[inline] // #[unstable(feature = "str_as_str", issue = "130366")] - #[unstable(feature = "bstr", issue = "134915")] + #[unstable(feature = "byte_str", issue = "134915")] pub const fn as_byte_str(&self) -> &ByteStr { self } @@ -88,15 +88,15 @@ impl ByteStr { /// for example `Box` or `MutexGuard`. #[inline] // #[unstable(feature = "str_as_str", issue = "130366")] - #[unstable(feature = "bstr", issue = "134915")] + #[unstable(feature = "byte_str", issue = "134915")] pub const fn as_mut_byte_str(&mut self) -> &mut ByteStr { self } #[doc(hidden)] - #[unstable(feature = "bstr_internals", issue = "none")] + #[unstable(feature = "byte_str_internals", issue = "none")] #[inline] - #[rustc_const_unstable(feature = "bstr_internals", issue = "none")] + #[rustc_const_unstable(feature = "byte_str_internals", issue = "none")] pub const fn from_bytes(slice: &[u8]) -> &Self { // SAFETY: `ByteStr` is a transparent wrapper around `[u8]`, so we can turn a reference to // the wrapped type into a reference to the wrapper type. @@ -104,9 +104,9 @@ impl ByteStr { } #[doc(hidden)] - #[unstable(feature = "bstr_internals", issue = "none")] + #[unstable(feature = "byte_str_internals", issue = "none")] #[inline] - #[rustc_const_unstable(feature = "bstr_internals", issue = "none")] + #[rustc_const_unstable(feature = "byte_str_internals", issue = "none")] pub const fn from_bytes_mut(slice: &mut [u8]) -> &mut Self { // SAFETY: `ByteStr` is a transparent wrapper around `[u8]`, so we can turn a reference to // the wrapped type into a reference to the wrapper type. @@ -114,23 +114,23 @@ impl ByteStr { } #[doc(hidden)] - #[unstable(feature = "bstr_internals", issue = "none")] + #[unstable(feature = "byte_str_internals", issue = "none")] #[inline] - #[rustc_const_unstable(feature = "bstr_internals", issue = "none")] + #[rustc_const_unstable(feature = "byte_str_internals", issue = "none")] pub const fn as_bytes(&self) -> &[u8] { &self.0 } #[doc(hidden)] - #[unstable(feature = "bstr_internals", issue = "none")] + #[unstable(feature = "byte_str_internals", issue = "none")] #[inline] - #[rustc_const_unstable(feature = "bstr_internals", issue = "none")] + #[rustc_const_unstable(feature = "byte_str_internals", issue = "none")] pub const fn as_bytes_mut(&mut self) -> &mut [u8] { &mut self.0 } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[rustc_const_unstable(feature = "const_convert", issue = "143773")] const impl Deref for ByteStr { type Target = [u8]; @@ -141,7 +141,7 @@ const impl Deref for ByteStr { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[rustc_const_unstable(feature = "const_convert", issue = "143773")] const impl DerefMut for ByteStr { #[inline] @@ -153,7 +153,7 @@ const impl DerefMut for ByteStr { #[unstable(feature = "deref_pure_trait", issue = "87121")] unsafe impl DerefPure for ByteStr {} -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl fmt::Debug for ByteStr { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "\"")?; @@ -172,7 +172,7 @@ impl fmt::Debug for ByteStr { } } -#[unstable(feature = "bstr_to_string", issue = "134915")] +#[unstable(feature = "byte_str_to_string", issue = "134915")] impl fmt::Display for ByteStr { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fn emit(byte_str: &ByteStr, f: &mut fmt::Formatter<'_>) -> fmt::Result { @@ -257,7 +257,7 @@ impl fmt::Display for ByteStr { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[rustc_const_unstable(feature = "const_convert", issue = "143773")] const impl AsRef<[u8]> for ByteStr { #[inline] @@ -266,7 +266,7 @@ const impl AsRef<[u8]> for ByteStr { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[rustc_const_unstable(feature = "const_convert", issue = "143773")] const impl AsRef for ByteStr { #[inline] @@ -277,7 +277,7 @@ const impl AsRef for ByteStr { // `impl AsRef for [u8]` omitted to avoid widespread inference failures -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[rustc_const_unstable(feature = "const_convert", issue = "143773")] const impl AsRef for str { #[inline] @@ -286,7 +286,7 @@ const impl AsRef for str { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[rustc_const_unstable(feature = "const_convert", issue = "143773")] const impl AsMut<[u8]> for ByteStr { #[inline] @@ -301,7 +301,7 @@ const impl AsMut<[u8]> for ByteStr { // `impl Borrow for str` omitted to avoid widespread inference failures -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[rustc_const_unstable(feature = "const_convert", issue = "143773")] const impl Borrow<[u8]> for ByteStr { #[inline] @@ -312,7 +312,7 @@ const impl Borrow<[u8]> for ByteStr { // `impl BorrowMut for [u8]` omitted to avoid widespread inference failures -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[rustc_const_unstable(feature = "const_convert", issue = "143773")] const impl BorrowMut<[u8]> for ByteStr { #[inline] @@ -321,14 +321,14 @@ const impl BorrowMut<[u8]> for ByteStr { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl<'a> Default for &'a ByteStr { fn default() -> Self { ByteStr::from_bytes(b"") } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl<'a> Default for &'a mut ByteStr { fn default() -> Self { ByteStr::from_bytes_mut(&mut []) @@ -337,7 +337,7 @@ impl<'a> Default for &'a mut ByteStr { // Omitted due to inference failures // -// #[unstable(feature = "bstr", issue = "134915")] +// #[unstable(feature = "byte_str", issue = "134915")] // impl<'a, const N: usize> From<&'a [u8; N]> for &'a ByteStr { // #[inline] // fn from(s: &'a [u8; N]) -> Self { @@ -345,7 +345,7 @@ impl<'a> Default for &'a mut ByteStr { // } // } // -// #[unstable(feature = "bstr", issue = "134915")] +// #[unstable(feature = "byte_str", issue = "134915")] // impl<'a> From<&'a [u8]> for &'a ByteStr { // #[inline] // fn from(s: &'a [u8]) -> Self { @@ -355,7 +355,7 @@ impl<'a> Default for &'a mut ByteStr { // Omitted due to slice-from-array-issue-113238: // -// #[unstable(feature = "bstr", issue = "134915")] +// #[unstable(feature = "byte_str", issue = "134915")] // impl<'a> From<&'a ByteStr> for &'a [u8] { // #[inline] // fn from(s: &'a ByteStr) -> Self { @@ -363,7 +363,7 @@ impl<'a> Default for &'a mut ByteStr { // } // } // -// #[unstable(feature = "bstr", issue = "134915")] +// #[unstable(feature = "byte_str", issue = "134915")] // impl<'a> From<&'a mut ByteStr> for &'a mut [u8] { // #[inline] // fn from(s: &'a mut ByteStr) -> Self { @@ -373,7 +373,7 @@ impl<'a> Default for &'a mut ByteStr { // Omitted due to inference failures // -// #[unstable(feature = "bstr", issue = "134915")] +// #[unstable(feature = "byte_str", issue = "134915")] // impl<'a> From<&'a str> for &'a ByteStr { // #[inline] // fn from(s: &'a str) -> Self { @@ -381,7 +381,7 @@ impl<'a> Default for &'a mut ByteStr { // } // } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[rustc_const_unstable(feature = "const_convert", issue = "143773")] const impl<'a> TryFrom<&'a ByteStr> for &'a str { type Error = crate::str::Utf8Error; @@ -392,7 +392,7 @@ const impl<'a> TryFrom<&'a ByteStr> for &'a str { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[rustc_const_unstable(feature = "const_convert", issue = "143773")] const impl<'a> TryFrom<&'a mut ByteStr> for &'a mut str { type Error = crate::str::Utf8Error; diff --git a/library/core/src/bstr/traits.rs b/library/core/src/byte_str/traits.rs similarity index 86% rename from library/core/src/bstr/traits.rs rename to library/core/src/byte_str/traits.rs index 6e9d836f1a1ff..93370ed7b0346 100644 --- a/library/core/src/bstr/traits.rs +++ b/library/core/src/byte_str/traits.rs @@ -1,11 +1,11 @@ //! Trait implementations for `ByteStr`. -use crate::bstr::ByteStr; +use crate::byte_str::ByteStr; use crate::cmp::Ordering; use crate::slice::SliceIndex; use crate::{hash, ops, range}; -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[rustc_const_unstable(feature = "const_cmp", issue = "143800")] const impl Ord for ByteStr { #[inline] @@ -14,7 +14,7 @@ const impl Ord for ByteStr { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[rustc_const_unstable(feature = "const_cmp", issue = "143800")] const impl PartialOrd for ByteStr { #[inline] @@ -23,7 +23,7 @@ const impl PartialOrd for ByteStr { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[rustc_const_unstable(feature = "const_cmp", issue = "143800")] const impl PartialEq for ByteStr { #[inline] @@ -32,11 +32,11 @@ const impl PartialEq for ByteStr { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] #[rustc_const_unstable(feature = "const_cmp", issue = "143800")] const impl Eq for ByteStr {} -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl hash::Hash for ByteStr { #[inline] fn hash(&self, state: &mut H) { @@ -46,7 +46,7 @@ impl hash::Hash for ByteStr { #[doc(hidden)] #[macro_export] -#[unstable(feature = "bstr_internals", issue = "none")] +#[unstable(feature = "byte_str_internals", issue = "none")] macro_rules! impl_partial_eq { ($lhs:ty, $rhs:ty) => { impl PartialEq<$rhs> for $lhs { @@ -68,17 +68,17 @@ macro_rules! impl_partial_eq { } #[doc(hidden)] -#[unstable(feature = "bstr_internals", issue = "none")] +#[unstable(feature = "byte_str_internals", issue = "none")] pub use impl_partial_eq; #[doc(hidden)] #[macro_export] -#[unstable(feature = "bstr_internals", issue = "none")] +#[unstable(feature = "byte_str_internals", issue = "none")] macro_rules! impl_partial_eq_ord { ($lhs:ty, $rhs:ty) => { - $crate::bstr::impl_partial_eq!($lhs, $rhs); + $crate::byte_str::impl_partial_eq!($lhs, $rhs); - #[unstable(feature = "bstr", issue = "134915")] + #[unstable(feature = "byte_str", issue = "134915")] impl PartialOrd<$rhs> for $lhs { #[inline] fn partial_cmp(&self, other: &$rhs) -> Option { @@ -87,7 +87,7 @@ macro_rules! impl_partial_eq_ord { } } - #[unstable(feature = "bstr", issue = "134915")] + #[unstable(feature = "byte_str", issue = "134915")] impl PartialOrd<$lhs> for $rhs { #[inline] fn partial_cmp(&self, other: &$lhs) -> Option { @@ -99,15 +99,15 @@ macro_rules! impl_partial_eq_ord { } #[doc(hidden)] -#[unstable(feature = "bstr_internals", issue = "none")] +#[unstable(feature = "byte_str_internals", issue = "none")] pub use impl_partial_eq_ord; #[doc(hidden)] #[macro_export] -#[unstable(feature = "bstr_internals", issue = "none")] +#[unstable(feature = "byte_str_internals", issue = "none")] macro_rules! impl_partial_eq_n { ($lhs:ty, $rhs:ty) => { - #[unstable(feature = "bstr", issue = "134915")] + #[unstable(feature = "byte_str", issue = "134915")] impl PartialEq<$rhs> for $lhs { #[inline] fn eq(&self, other: &$rhs) -> bool { @@ -116,7 +116,7 @@ macro_rules! impl_partial_eq_n { } } - #[unstable(feature = "bstr", issue = "134915")] + #[unstable(feature = "byte_str", issue = "134915")] impl PartialEq<$lhs> for $rhs { #[inline] fn eq(&self, other: &$lhs) -> bool { @@ -128,7 +128,7 @@ macro_rules! impl_partial_eq_n { } #[doc(hidden)] -#[unstable(feature = "bstr_internals", issue = "none")] +#[unstable(feature = "byte_str_internals", issue = "none")] pub use impl_partial_eq_n; // PartialOrd with `[u8]` omitted to avoid inference failures @@ -144,7 +144,7 @@ impl_partial_eq_n!(ByteStr, [u8; N]); // PartialOrd with `[u8; N]` omitted to avoid inference failures impl_partial_eq_n!(ByteStr, &[u8; N]); -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl ops::Index for ByteStr where I: SliceIndex, @@ -157,7 +157,7 @@ where } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] impl ops::IndexMut for ByteStr where I: SliceIndex, @@ -168,7 +168,7 @@ where } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] unsafe impl SliceIndex for ops::RangeFull { type Output = ByteStr; #[inline] @@ -197,7 +197,7 @@ unsafe impl SliceIndex for ops::RangeFull { } } -#[unstable(feature = "bstr", issue = "134915")] +#[unstable(feature = "byte_str", issue = "134915")] unsafe impl SliceIndex for usize { type Output = u8; #[inline] @@ -230,7 +230,7 @@ unsafe impl SliceIndex for usize { macro_rules! impl_slice_index { ($index:ty) => { - #[unstable(feature = "bstr", issue = "134915")] + #[unstable(feature = "byte_str", issue = "134915")] unsafe impl SliceIndex for $index { type Output = ByteStr; #[inline] diff --git a/library/core/src/cell.rs b/library/core/src/cell.rs index 3dcf033bbd757..d37f0eaa512a6 100644 --- a/library/core/src/cell.rs +++ b/library/core/src/cell.rs @@ -312,7 +312,7 @@ pub use once::OnceCell; #[stable(feature = "rust1", since = "1.0.0")] #[repr(transparent)] #[rustc_pub_transparent] -pub struct Cell { +pub struct Cell<#[rustc_assert_variance(invariant)] T: ?Sized> { value: UnsafeCell, } @@ -849,7 +849,7 @@ impl Cell { /// See the [module-level documentation](self) for more. #[rustc_diagnostic_item = "RefCell"] #[stable(feature = "rust1", since = "1.0.0")] -pub struct RefCell { +pub struct RefCell<#[rustc_assert_variance(invariant)] T: ?Sized> { borrow: Cell, // Stores the location of the earliest currently active borrow. // This gets updated whenever we go from having zero borrows @@ -2308,7 +2308,7 @@ impl fmt::Display for RefMut<'_, T> { #[stable(feature = "rust1", since = "1.0.0")] #[repr(transparent)] #[rustc_pub_transparent] -pub struct UnsafeCell { +pub struct UnsafeCell<#[rustc_assert_variance(invariant)] T: ?Sized> { value: T, } @@ -2603,7 +2603,7 @@ impl, U> DispatchFromDyn> for UnsafeCell #[repr(transparent)] #[rustc_diagnostic_item = "SyncUnsafeCell"] #[rustc_pub_transparent] -pub struct SyncUnsafeCell { +pub struct SyncUnsafeCell<#[rustc_assert_variance(invariant)] T: ?Sized> { value: UnsafeCell, } diff --git a/library/core/src/cell/covariant_unsafe_cell.rs b/library/core/src/cell/covariant_unsafe_cell.rs index 11ac9b58bbb90..bd38fb303d914 100644 --- a/library/core/src/cell/covariant_unsafe_cell.rs +++ b/library/core/src/cell/covariant_unsafe_cell.rs @@ -17,7 +17,7 @@ use crate::ptr::{self, NonNull}; // hardcode its variance to be **co**variant in `T`, even though it is wrapping `UnsafeCell` which // is **in**variant in `T`. #[lang = "covariant_unsafe_cell"] -pub struct CovariantUnsafeCell(UnsafeCell); +pub struct CovariantUnsafeCell<#[rustc_assert_variance(covariant)] T: ?Sized>(UnsafeCell); #[unstable(feature = "covariant_unsafe_cell", issue = "159735")] impl !Sync for CovariantUnsafeCell {} diff --git a/library/core/src/cell/lazy.rs b/library/core/src/cell/lazy.rs index ba32fff2284fd..fc96376cbe70d 100644 --- a/library/core/src/cell/lazy.rs +++ b/library/core/src/cell/lazy.rs @@ -51,7 +51,10 @@ enum State { /// // 92 /// ``` #[stable(feature = "lazy_cell", since = "1.80.0")] -pub struct LazyCell T> { +pub struct LazyCell< + #[rustc_assert_variance(invariant)] T, + #[rustc_assert_variance(invariant)] F = fn() -> T, +> { state: UnsafeCell>, } diff --git a/library/core/src/cell/once.rs b/library/core/src/cell/once.rs index 01ba604e67949..6df10de2d5aae 100644 --- a/library/core/src/cell/once.rs +++ b/library/core/src/cell/once.rs @@ -32,7 +32,7 @@ use crate::{fmt, mem}; /// assert!(cell.get().is_some()); /// ``` #[stable(feature = "once_cell", since = "1.70.0")] -pub struct OnceCell { +pub struct OnceCell<#[rustc_assert_variance(invariant)] T> { // Invariant: written to at most once. inner: UnsafeCell>, } diff --git a/library/core/src/clone.rs b/library/core/src/clone.rs index bd6d929c6d2b0..61528873a38c7 100644 --- a/library/core/src/clone.rs +++ b/library/core/src/clone.rs @@ -686,8 +686,8 @@ unsafe impl CloneToUninit for crate::ffi::CStr { } } -#[unstable(feature = "bstr", issue = "134915")] -unsafe impl CloneToUninit for crate::bstr::ByteStr { +#[unstable(feature = "byte_str", issue = "134915")] +unsafe impl CloneToUninit for crate::byte_str::ByteStr { #[inline] #[cfg_attr(debug_assertions, track_caller)] unsafe fn clone_to_uninit(&self, dst: *mut u8) { diff --git a/library/core/src/cmp.rs b/library/core/src/cmp.rs index 1b57bed2738f7..83a2925678e19 100644 --- a/library/core/src/cmp.rs +++ b/library/core/src/cmp.rs @@ -1557,8 +1557,10 @@ pub const trait PartialOrd: self.partial_cmp(other).is_some_and(Ordering::is_ge) } - /// If `self == other`, returns `ControlFlow::Continue(())`. - /// Otherwise, returns `ControlFlow::Break(self < other)`. + /// If `self == other`, returns `Continue(())`. + /// Otherwise, returns `Break(self < other)`, or equivalently `Break(self <= other)`. + /// The only difference when chaining `<` and `<=` is how an equality result is + /// interpreted at the end of the chain. /// /// This is useful for chaining together calls when implementing a lexical /// `PartialOrd::lt`, as it allows types (like primitives) which can cheaply @@ -1568,33 +1570,17 @@ pub const trait PartialOrd: // Added to improve the behaviour of tuples; not necessarily stabilization-track. #[unstable(feature = "partial_ord_chaining_methods", issue = "none")] #[doc(hidden)] - fn __chaining_lt(&self, other: &Rhs) -> ControlFlow { + fn __chaining_less(&self, other: &Rhs) -> ControlFlow { default_chaining_impl(self, other, Ordering::is_lt) } - /// Same as `__chaining_lt`, but for `<=` instead of `<`. + /// Same as `__chaining_less`, but for `>`/`>=` instead of `<`/`<=`. #[inline] #[unstable(feature = "partial_ord_chaining_methods", issue = "none")] #[doc(hidden)] - fn __chaining_le(&self, other: &Rhs) -> ControlFlow { - default_chaining_impl(self, other, Ordering::is_le) - } - - /// Same as `__chaining_lt`, but for `>` instead of `<`. - #[inline] - #[unstable(feature = "partial_ord_chaining_methods", issue = "none")] - #[doc(hidden)] - fn __chaining_gt(&self, other: &Rhs) -> ControlFlow { + fn __chaining_greater(&self, other: &Rhs) -> ControlFlow { default_chaining_impl(self, other, Ordering::is_gt) } - - /// Same as `__chaining_lt`, but for `>=` instead of `<`. - #[inline] - #[unstable(feature = "partial_ord_chaining_methods", issue = "none")] - #[doc(hidden)] - fn __chaining_ge(&self, other: &Rhs) -> ControlFlow { - default_chaining_impl(self, other, Ordering::is_ge) - } } #[rustc_const_unstable(feature = "const_cmp", issue = "143800")] @@ -2225,25 +2211,17 @@ mod impls { // the `Break` case and the comparison will correctly return `false`. #[inline] - fn __chaining_lt(&self, other: &Self) -> ControlFlow { + fn __chaining_less(&self, other: &Self) -> ControlFlow { let (lhs, rhs) = (*self, *other); + // We arbitrarily pick `<` here. Using `<=` gives the same result. if lhs == rhs { Continue(()) } else { Break(lhs < rhs) } } #[inline] - fn __chaining_le(&self, other: &Self) -> ControlFlow { - let (lhs, rhs) = (*self, *other); - if lhs == rhs { Continue(()) } else { Break(lhs <= rhs) } - } - #[inline] - fn __chaining_gt(&self, other: &Self) -> ControlFlow { + fn __chaining_greater(&self, other: &Self) -> ControlFlow { let (lhs, rhs) = (*self, *other); + // We arbitrarily pick `>` here. Using `>=` gives the same result. if lhs == rhs { Continue(()) } else { Break(lhs > rhs) } } - #[inline] - fn __chaining_ge(&self, other: &Self) -> ControlFlow { - let (lhs, rhs) = (*self, *other); - if lhs == rhs { Continue(()) } else { Break(lhs >= rhs) } - } }; } @@ -2484,20 +2462,12 @@ mod impls { PartialOrd::ge(*self, *other) } #[inline] - fn __chaining_lt(&self, other: &&B) -> ControlFlow { - PartialOrd::__chaining_lt(*self, *other) - } - #[inline] - fn __chaining_le(&self, other: &&B) -> ControlFlow { - PartialOrd::__chaining_le(*self, *other) + fn __chaining_less(&self, other: &&B) -> ControlFlow { + PartialOrd::__chaining_less(*self, *other) } #[inline] - fn __chaining_gt(&self, other: &&B) -> ControlFlow { - PartialOrd::__chaining_gt(*self, *other) - } - #[inline] - fn __chaining_ge(&self, other: &&B) -> ControlFlow { - PartialOrd::__chaining_ge(*self, *other) + fn __chaining_greater(&self, other: &&B) -> ControlFlow { + PartialOrd::__chaining_greater(*self, *other) } } #[stable(feature = "rust1", since = "1.0.0")] @@ -2559,20 +2529,12 @@ mod impls { PartialOrd::ge(*self, *other) } #[inline] - fn __chaining_lt(&self, other: &&mut B) -> ControlFlow { - PartialOrd::__chaining_lt(*self, *other) - } - #[inline] - fn __chaining_le(&self, other: &&mut B) -> ControlFlow { - PartialOrd::__chaining_le(*self, *other) - } - #[inline] - fn __chaining_gt(&self, other: &&mut B) -> ControlFlow { - PartialOrd::__chaining_gt(*self, *other) + fn __chaining_less(&self, other: &&mut B) -> ControlFlow { + PartialOrd::__chaining_less(*self, *other) } #[inline] - fn __chaining_ge(&self, other: &&mut B) -> ControlFlow { - PartialOrd::__chaining_ge(*self, *other) + fn __chaining_greater(&self, other: &&mut B) -> ControlFlow { + PartialOrd::__chaining_greater(*self, *other) } } #[stable(feature = "rust1", since = "1.0.0")] diff --git a/library/core/src/convert/mod.rs b/library/core/src/convert/mod.rs index b0f74a8afb598..c86c70b5c4ca1 100644 --- a/library/core/src/convert/mod.rs +++ b/library/core/src/convert/mod.rs @@ -863,6 +863,28 @@ const impl AsMut for str { } } +#[stable(feature = "never_type", since = "1.100.0")] +#[expect( + clippy::explicit_auto_deref, + reason = "https://github.com/rust-lang/rust-clippy/issues/17713" +)] +impl AsRef for ! { + fn as_ref(&self) -> &T { + *self + } +} + +#[stable(feature = "never_type", since = "1.100.0")] +#[expect( + clippy::explicit_auto_deref, + reason = "https://github.com/rust-lang/rust-clippy/issues/17713" +)] +impl AsMut for ! { + fn as_mut(&mut self) -> &mut T { + *self + } +} + //////////////////////////////////////////////////////////////////////////////// // THE NO-ERROR ERROR TYPE //////////////////////////////////////////////////////////////////////////////// diff --git a/library/core/src/ffi/c_str.rs b/library/core/src/ffi/c_str.rs index e299f44e03181..eb661ef7b8460 100644 --- a/library/core/src/ffi/c_str.rs +++ b/library/core/src/ffi/c_str.rs @@ -180,7 +180,7 @@ impl fmt::Display for FromBytesUntilNulError { #[stable(feature = "cstr_debug", since = "1.3.0")] impl fmt::Debug for CStr { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Debug::fmt(crate::bstr::ByteStr::from_bytes(self.to_bytes()), f) + fmt::Debug::fmt(crate::byte_str::ByteStr::from_bytes(self.to_bytes()), f) } } @@ -942,7 +942,7 @@ impl fmt::Debug for Display<'_> { #[stable(feature = "cstr_display", since = "CURRENT_RUSTC_VERSION")] impl fmt::Display for Display<'_> { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - fmt::Display::fmt(crate::bstr::ByteStr::from_bytes(self.c_str.to_bytes()), f) + fmt::Display::fmt(crate::byte_str::ByteStr::from_bytes(self.c_str.to_bytes()), f) } } diff --git a/library/core/src/ffi/va_list.rs b/library/core/src/ffi/va_list.rs index b308cb7f85df2..74213005b566d 100644 --- a/library/core/src/ffi/va_list.rs +++ b/library/core/src/ffi/va_list.rs @@ -465,6 +465,20 @@ cfg_select! { _ => { /* unsupported */ } } +cfg_select! { + target_arch = "powerpc" => { + #[unstable_feature_bound(powerpc_ppcf128)] + #[unstable(feature = "powerpc_ppcf128", issue = "163323")] + unsafe impl VaArgSafe for crate::arch::powerpc::ppcf128 {} + } + target_arch = "powerpc64" => { + #[unstable_feature_bound(powerpc_ppcf128)] + #[unstable(feature = "powerpc_ppcf128", issue = "163323")] + unsafe impl VaArgSafe for crate::arch::powerpc64::ppcf128 {} + } + _ => { /* ppcf128 is not defined */ } +} + #[unstable(feature = "c_variadic_va_arg_safe", issue = "162911", implied_by = "c_variadic")] unsafe impl VaArgSafe for *mut T {} #[unstable(feature = "c_variadic_va_arg_safe", issue = "162911", implied_by = "c_variadic")] diff --git a/library/core/src/fmt/mod.rs b/library/core/src/fmt/mod.rs index 1f1b581c5e546..caa150cf88364 100644 --- a/library/core/src/fmt/mod.rs +++ b/library/core/src/fmt/mod.rs @@ -1038,14 +1038,14 @@ impl Display for Arguments<'_> { #[rustc_on_unimplemented( on( all(crate_local, not(Self = "{union}")), - note = "add `#[derive(Debug)]` to `{Self}` or manually `impl {This} for {Self}`" + note = "add `#[derive(Debug)]` to `{Self}` or manually `impl {this} for {Self}`" ), - on(all(crate_local, Self = "{union}"), note = "manually `impl {This} for {Self}`"), + on(all(crate_local, Self = "{union}"), note = "manually `impl {this} for {Self}`"), on( from_desugaring = "FormatLiteral", - label = "`{Self}` cannot be formatted using `{{:?}}` because it doesn't implement `{This}`" + label = "`{Self}` cannot be formatted using `{{:?}}` because it doesn't implement `{this}`" ), - message = "`{Self}` doesn't implement `{This}`" + message = "`{Self}` doesn't implement `{this}`" )] #[doc(alias = "{:?}")] #[rustc_diagnostic_item = "Debug"] @@ -1179,7 +1179,7 @@ pub use macros::Debug; note = "in format strings you may be able to use `{{:?}}` (or `{{:#?}}` for pretty-print) instead", label = "`{Self}` cannot be formatted with the default formatter", ), - message = "`{Self}` doesn't implement `{This}`" + message = "`{Self}` doesn't implement `{this}`" )] #[doc(alias = "{}")] #[rustc_diagnostic_item = "Display"] @@ -2107,6 +2107,10 @@ impl<'a> Formatter<'a> { /// /// Writes some formatted information into this instance. /// + /// This method delegates to the underlying [`Write`] implementation, so + /// whether the formatted output is appended to, or replaces, previously + /// written data is up to that writer. + /// /// # Examples /// /// ``` @@ -2123,6 +2127,44 @@ impl<'a> Formatter<'a> { /// assert_eq!(format!("{}", Foo(-1)), "Foo -1"); /// assert_eq!(format!("{:0>8}", Foo(2)), "Foo 2"); /// ``` + /// + /// The same formatting code can produce different output depending on the + /// underlying writer: + /// + /// ``` + /// use std::fmt; + /// + /// struct Hello; + /// + /// impl fmt::Display for Hello { + /// fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + /// formatter.write_fmt(format_args!("Hello"))?; + /// formatter.write_fmt(format_args!(" world")) + /// } + /// } + /// + /// // A writer that replaces its contents on every write. + /// struct Replace(String); + /// + /// impl fmt::Write for Replace { + /// fn write_str(&mut self, s: &str) -> fmt::Result { + /// self.0.clear(); + /// self.0.push_str(s); + /// Ok(()) + /// } + /// } + /// + /// // With `String`, which appends, both writes are kept. + /// let mut s = String::new(); + /// fmt::write(&mut s, format_args!("{Hello}"))?; + /// assert_eq!(s, "Hello world"); + /// + /// // With a writer that replaces, the second write replaces the first. + /// let mut w = Replace(String::new()); + /// fmt::write(&mut w, format_args!("{Hello}"))?; + /// assert_eq!(w.0, " world"); + /// # Ok::<(), fmt::Error>(()) + /// ``` #[stable(feature = "rust1", since = "1.0.0")] #[inline] pub fn write_fmt(&mut self, fmt: Arguments<'_>) -> Result { diff --git a/library/core/src/hash/mod.rs b/library/core/src/hash/mod.rs index f1a93a880e7f4..dbec6035e5217 100644 --- a/library/core/src/hash/mod.rs +++ b/library/core/src/hash/mod.rs @@ -436,7 +436,7 @@ pub trait Hasher { /// writing its contents to this `Hasher`. That way /// `(collection![1, 2, 3], collection![4, 5])` and /// `(collection![1, 2], collection![3, 4, 5])` will provide different - /// sequences of values to the `Hasher` + /// sequences of values to the `Hasher`. /// /// The `impl Hash for [T]` includes a call to this method, so if you're /// hashing a slice (or array or vector) via its `Hash::hash` method, diff --git a/library/core/src/hint.rs b/library/core/src/hint.rs index 73ef90d269f74..443e929e52cd3 100644 --- a/library/core/src/hint.rs +++ b/library/core/src/hint.rs @@ -710,7 +710,7 @@ pub const fn likely(b: bool) -> bool { /// // Use outside of a branch condition may still influence a nearby branch /// let cond = unlikely(x != 0); /// if cond { -/// println!("this branch is likely to be taken"); +/// println!("this branch is unlikely to be taken"); /// } /// } /// ``` diff --git a/library/core/src/intrinsics/mod.rs b/library/core/src/intrinsics/mod.rs index 391fcefbe3f05..a12705189fb04 100644 --- a/library/core/src/intrinsics/mod.rs +++ b/library/core/src/intrinsics/mod.rs @@ -577,6 +577,30 @@ pub const fn prefetch_write_instruction(data: *const T) #[rustc_nounwind] pub fn breakpoint(); +/// A container for the string arguments passed to [`codeview_annotation`]. +#[unstable(feature = "codeview_annotation", issue = "163964")] +pub trait CodeViewAnnotationArgs { + /// The string arguments to `codeview_annotation`. + const ARGS: &[&str]; +} + +/// Emits a call to [`llvm.codeview.annotation`](https://llvm.org/docs/LangRef.html#llvm-codeview-annotation-intrinsic) +/// which results in [`CodeViewAnnotationArgs::ARGS`] being written to the +/// PDB as an `S_ANNOTATION` record. +/// +/// Works only with targets that use PDB debuginfo and with the LLVM backend. +/// Is a no-op on other targets and backends. +#[unstable(feature = "codeview_annotation", issue = "163964")] +#[rustc_intrinsic] +#[rustc_nounwind] +#[miri::intrinsic_fallback_is_spec] +pub fn codeview_annotation() { + // const block to catch any errors in `T::ARGS` + const { + let _ignore = T::ARGS; + } +} + /// Magic intrinsic that derives its meaning from attributes /// attached to the function. /// diff --git a/library/core/src/iter/adapters/skip.rs b/library/core/src/iter/adapters/skip.rs index 9173489531107..f81df1a360e2e 100644 --- a/library/core/src/iter/adapters/skip.rs +++ b/library/core/src/iter/adapters/skip.rs @@ -139,6 +139,14 @@ where #[inline] #[rustc_inherit_overflow_checks] fn advance_by(&mut self, mut n: usize) -> Result<(), NonZero> { + if self.n > 0 && n == 0 { + // If we try to process our skips now, we might witness the end of the inner iterator, + // but there's no valid `Err(k)` that we could return. This is especially a problem for + // unfused iterators that might end up called again by `Chain`, for example, so it's + // better that we do nothing at all. + return Ok(()); + } + let skip_inner = self.n; let skip_and_advance = skip_inner.saturating_add(n); diff --git a/library/core/src/keyword_docs.rs b/library/core/src/keyword_docs.rs index 2ea028ef8439d..c2fcb790a3e68 100644 --- a/library/core/src/keyword_docs.rs +++ b/library/core/src/keyword_docs.rs @@ -1295,7 +1295,7 @@ const _: () = (); /// /// Example of using `become` to implement functional-style `fold`: /// -/// ```ignore-wasm (tail-call target feature not enabled by default on wasm) +/// ```ignore-wasm,ignore-powerpc,ignore-powerpc64 (tail-call target feature not enabled by default on wasm, PowerPC tail calls are not implemented in LLVM) /// #![feature(explicit_tail_calls)] /// #![expect(incomplete_features)] /// @@ -1350,7 +1350,7 @@ const _: () = (); /// /// It is possible to tail-call a function pointer: /// -/// ```ignore-wasm (tail-call target feature not enabled by default on wasm) +/// ```ignore-wasm,ignore-powerpc,ignore-powerpc64 (tail-call target feature not enabled by default on wasm, PowerPC tail calls are not implemented in LLVM) /// #![feature(explicit_tail_calls)] /// #![expect(incomplete_features)] /// diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index 350a348cc0c80..05f61eb68f667 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -86,7 +86,7 @@ // Library features: // tidy-alphabetical-start #![feature(asm_experimental_arch)] -#![feature(bstr_internals)] +#![feature(byte_str_internals)] #![feature(cfg_target_has_reliable_f16_f128)] #![feature(const_carrying_mul_add)] #![feature(const_cmp)] @@ -297,8 +297,8 @@ pub mod ascii; pub mod asserting; #[unstable(feature = "async_iterator", issue = "79024")] pub mod async_iter; -#[unstable(feature = "bstr", issue = "134915")] -pub mod bstr; +#[unstable(feature = "byte_str", issue = "134915")] +pub mod byte_str; pub mod cell; pub mod char; pub mod ffi; diff --git a/library/core/src/marker.rs b/library/core/src/marker.rs index 83dbd7d994367..c13e44d2f3ef5 100644 --- a/library/core/src/marker.rs +++ b/library/core/src/marker.rs @@ -1137,7 +1137,7 @@ marker_impls! { } marker_impls! { - #[unstable(feature = "unsized_const_params", issue = "95174")] + #[unstable(feature = "unsized_const_params", issue = "128028")] #[unstable_feature_bound(unsized_const_params)] ConstParamTy_ for str, diff --git a/library/core/src/marker/variance.rs b/library/core/src/marker/variance.rs index 9e135e89c451e..67dcdb5513027 100644 --- a/library/core/src/marker/variance.rs +++ b/library/core/src/marker/variance.rs @@ -16,10 +16,10 @@ macro_rules! first_token { macro_rules! phantom_type { ($( $(#[$attr:meta])* - pub struct $name:ident <$t:ident> ($($inner:tt)*); + pub struct $name:ident <$(#[$param_attr:meta])? $t:ident> ($($inner:tt)*); )*) => {$( $(#[$attr])* - pub struct $name<$t>($($inner)*) where $t: ?Sized; + pub struct $name<$(#[$param_attr])? $t>($($inner)*) where $t: ?Sized; impl $name where T: ?Sized @@ -101,11 +101,11 @@ macro_rules! phantom_type { macro_rules! phantom_lifetime { ($( $(#[$attr:meta])* - pub struct $name:ident <$lt:lifetime> ($($inner:tt)*); + pub struct $name:ident <$(#[$param_attr:meta])? $lt:lifetime> ($($inner:tt)*); )*) => {$( $(#[$attr])* #[derive(Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] - pub struct $name<$lt>($($inner)*); + pub struct $name<$(#[$param_attr])? $lt>($($inner)*); impl $name<'_> { /// Constructs a new instance of the variance marker. @@ -144,7 +144,7 @@ phantom_lifetime! { /// * `align_of::>() == 1` #[rustc_pub_transparent] #[repr(transparent)] - pub struct PhantomCovariantLifetime<'a>(PhantomCovariant<&'a ()>); + pub struct PhantomCovariantLifetime<#[rustc_assert_variance(covariant)] 'a>(PhantomCovariant<&'a ()>); /// Zero-sized type used to mark a lifetime as contravariant. /// /// Contravariant lifetimes must live at most as long as declared. See [the reference][1] for @@ -161,7 +161,7 @@ phantom_lifetime! { /// * `align_of::>() == 1` #[rustc_pub_transparent] #[repr(transparent)] - pub struct PhantomContravariantLifetime<'a>(PhantomContravariant<&'a ()>); + pub struct PhantomContravariantLifetime<#[rustc_assert_variance(contravariant)] 'a>(PhantomContravariant<&'a ()>); /// Zero-sized type used to mark a lifetime as invariant. /// /// Invariant lifetimes must be live for the exact length declared, neither shorter nor longer. @@ -176,7 +176,7 @@ phantom_lifetime! { /// * `align_of::>() == 1` #[rustc_pub_transparent] #[repr(transparent)] - pub struct PhantomInvariantLifetime<'a>(PhantomInvariant<&'a ()>); + pub struct PhantomInvariantLifetime<#[rustc_assert_variance(invariant)] 'a>(PhantomInvariant<&'a ()>); } phantom_type! { @@ -197,7 +197,7 @@ phantom_type! { /// * `align_of::>() == 1` #[rustc_pub_transparent] #[repr(transparent)] - pub struct PhantomCovariant(PhantomData T>); + pub struct PhantomCovariant<#[rustc_assert_variance(covariant)] T>(PhantomData T>); /// Zero-sized type used to mark a type parameter as contravariant. /// /// Types passed as arguments to a function are contravariant. If the type is _also_ part of the @@ -215,7 +215,7 @@ phantom_type! { /// * `align_of::>() == 1` #[rustc_pub_transparent] #[repr(transparent)] - pub struct PhantomContravariant(PhantomData); + pub struct PhantomContravariant<#[rustc_assert_variance(contravariant)] T>(PhantomData); /// Zero-sized type used to mark a type parameter as invariant. /// /// Types that are both passed as an argument _and_ used as part of the return value from a @@ -230,7 +230,7 @@ phantom_type! { /// * `align_of::>() == 1` #[rustc_pub_transparent] #[repr(transparent)] - pub struct PhantomInvariant(PhantomData T>); + pub struct PhantomInvariant<#[rustc_assert_variance(invariant)] T>(PhantomData T>); } mod private_items { diff --git a/library/core/src/mem/drop_guard.rs b/library/core/src/mem/drop_guard.rs index 81104ccce29f1..d997ab680e354 100644 --- a/library/core/src/mem/drop_guard.rs +++ b/library/core/src/mem/drop_guard.rs @@ -30,7 +30,7 @@ use crate::ops::{Deref, DerefMut}; #[stable(feature = "drop_guard", since = "1.100.0")] #[doc(alias = "ScopeGuard")] #[doc(alias = "defer")] -pub struct DropGuard +pub struct DropGuard<#[rustc_assert_variance(covariant)] T, #[rustc_assert_variance(covariant)] F> where F: FnOnce(T), { diff --git a/library/core/src/net/ip_addr.rs b/library/core/src/net/ip_addr.rs index f09410b18651e..aeb3ceab25e05 100644 --- a/library/core/src/net/ip_addr.rs +++ b/library/core/src/net/ip_addr.rs @@ -925,6 +925,8 @@ impl Ipv4Addr { self.octets()[0] == 192 && self.octets()[1] == 0 && self.octets()[2] == 0 && self.octets()[3] != 9 && self.octets()[3] != 10 ) + // 6a44-relay anycast address (`192.88.99.2/32`) + || matches!(self.octets(), [192, 88, 99, 2]) || self.is_documentation() || self.is_benchmarking() || self.is_reserved() @@ -1690,6 +1692,8 @@ impl Ipv6Addr { || matches!(self.segments(), [0x64, 0xff9b, 1, _, _, _, _, _]) // Discard-Only Address Block (`100::/64`) || matches!(self.segments(), [0x100, 0, 0, 0, _, _, _, _]) + // Dummy IPv6 Prefix (100:0:0:1::/64) + || matches!(self.segments(), [0x100, 0, 0, 1, _, _, _, _]) // IETF Protocol Assignments (`2001::/23`) || (matches!(self.segments(), [0x2001, b, _, _, _, _, _, _] if b < 0x200) && !( @@ -1697,6 +1701,8 @@ impl Ipv6Addr { u128::from_be_bytes(self.octets()) == 0x2001_0001_0000_0000_0000_0000_0000_0001 // Traversal Using Relays around NAT Anycast (`2001:1::2`) || u128::from_be_bytes(self.octets()) == 0x2001_0001_0000_0000_0000_0000_0000_0002 + // DNS-SD Service Registration Protocol Anycast (`2001:1::3`) + || u128::from_be_bytes(self.octets()) == 0x2001_0001_0000_0000_0000_0000_0000_0003 // AMT (`2001:3::/32`) || matches!(self.segments(), [0x2001, 3, _, _, _, _, _, _]) // AS112-v6 (`2001:4:112::/48`) diff --git a/library/core/src/num/imp/int_bits.rs b/library/core/src/num/imp/int_bits.rs index 2bc95e66e7ef6..dbb1a54fc59f6 100644 --- a/library/core/src/num/imp/int_bits.rs +++ b/library/core/src/num/imp/int_bits.rs @@ -115,7 +115,7 @@ macro_rules! uint_impl { // [0; n], [X; a], [0; n], [0; b] // q // [0; n], [0; a + n], [Y; b] // x ^= q // [0; n + n], [X; a], [0; b] // q >> n - // [0; n], [0; n], [X; a], [Y; b] // x ^= q << n + // [0; n], [0; n], [X; a], [Y; b] // x ^= q >> n // ``` // Only zeros are shifted out, satisfying the assumption // for the next group. @@ -123,9 +123,18 @@ macro_rules! uint_impl { // In effect, the upper run of data is swapped with the // group of `n` zeros below it. let q = x & masks[stage]; - x ^= q; - x ^= q >> n; + if cfg!(all(target_arch = "aarch64", target_feature = "neon")) && $U::BITS <= 64 + { + // `q >> n` has no set bits in common with `x ^ q`, so addition + // is equivalent to XOR here. For widths up to 64 bits this + // allows AArch64 to use `usra`. For `u128`, this add may introduce + // a slower cross-limb carry chain. + x = (x & !masks[stage]).wrapping_add(q >> n); + } else { + x ^= q; + x ^= q >> n; + } stage += 1; } x diff --git a/library/core/src/offload/mod.rs b/library/core/src/offload/mod.rs index 3d85621361209..aefe3a4eb164d 100644 --- a/library/core/src/offload/mod.rs +++ b/library/core/src/offload/mod.rs @@ -1,4 +1,6 @@ // offload module +use core::ptr::NonNull; + #[unstable(feature = "gpu_offload", issue = "131513")] pub use crate::macros::builtin::offload_kernel; #[unstable(feature = "gpu_offload", issue = "131513")] @@ -37,58 +39,52 @@ pub use crate::offload; /// ``` #[macro_export] #[unstable(feature = "gpu_offload", issue = "131513")] -#[allow_internal_unstable(core_intrinsics)] +#[allow_internal_unstable(core_intrinsics, gpu_offload)] macro_rules! offload { - ( $($field:ident = $val:expr),* $(,)? ) => { - $crate::offload!(@munch - [ $($field = $val),* ]; - kernel = NONE; - workgroup_dim = ([1, 1, 1]); - thread_dim = ([1, 1, 1]); - dyn_cache = (0); - device = NONE; - args = NONE - ) - }; - - (@munch [kernel = $val:expr $(, $rest_f:ident = $rest_v:expr)*]; kernel = NONE; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = $a:tt) => { - $crate::offload!(@munch [$($rest_f = $rest_v),*]; kernel = (SOME $val); workgroup_dim = $w; thread_dim = $t; dyn_cache = $d; device = $device; args = $a) + (@munch [kernel = $val:expr $(, $($rest:tt)*)?]; kernel = NONE; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = $a:tt) => { + $crate::offload!(@munch [ $($($rest)*)? ]; kernel = (SOME $val); workgroup_dim = $w; thread_dim = $t; dyn_cache = $d; device = $device; args = $a) }; - (@munch [kernel = $val:expr $(, $rest_f:ident = $rest_v:expr)*]; kernel = (SOME $old:expr); workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = $a:tt) => { + (@munch [kernel = $val:expr $(, $($rest:tt)*)?]; kernel = (SOME $old:expr); workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = $a:tt) => { compile_error!("duplicate field `kernel`") }; - (@munch [workgroup_dim = $val:expr $(, $rest_f:ident = $rest_v:expr)*]; kernel = $k:tt; workgroup_dim = ([1, 1, 1]); thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = $a:tt) => { - $crate::offload!(@munch [$($rest_f = $rest_v),*]; kernel = $k; workgroup_dim = (SOME $val); thread_dim = $t; dyn_cache = $d; device = $device; args = $a) + (@munch [workgroup_dim = $val:expr $(, $($rest:tt)*)?]; kernel = $k:tt; workgroup_dim = ([1, 1, 1]); thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = $a:tt) => { + $crate::offload!(@munch [ $($($rest)*)? ]; kernel = $k; workgroup_dim = (SOME $val); thread_dim = $t; dyn_cache = $d; device = $device; args = $a) }; - (@munch [workgroup_dim = $val:expr $(, $rest_f:ident = $rest_v:expr)*]; kernel = $k:tt; workgroup_dim = (SOME $old:expr); thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = $a:tt) => { + (@munch [workgroup_dim = $val:expr $(, $($rest:tt)*)?]; kernel = $k:tt; workgroup_dim = (SOME $old:expr); thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = $a:tt) => { compile_error!("duplicate field `workgroup_dim`") }; - (@munch [thread_dim = $val:expr $(, $rest_f:ident = $rest_v:expr)*]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = ([1, 1, 1]); dyn_cache = $d:tt; device = $device:tt; args = $a:tt) => { - $crate::offload!(@munch [$($rest_f = $rest_v),*]; kernel = $k; workgroup_dim = $w; thread_dim = (SOME $val); dyn_cache = $d; device = $device; args = $a) + (@munch [thread_dim = $val:expr $(, $($rest:tt)*)?]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = ([1, 1, 1]); dyn_cache = $d:tt; device = $device:tt; args = $a:tt) => { + $crate::offload!(@munch [ $($($rest)*)? ]; kernel = $k; workgroup_dim = $w; thread_dim = (SOME $val); dyn_cache = $d; device = $device; args = $a) }; - (@munch [thread_dim = $val:expr $(, $rest_f:ident = $rest_v:expr)*]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = (SOME $old:expr); dyn_cache = $d:tt; device = $device:tt; args = $a:tt) => { + (@munch [thread_dim = $val:expr $(, $($rest:tt)*)?]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = (SOME $old:expr); dyn_cache = $d:tt; device = $device:tt; args = $a:tt) => { compile_error!("duplicate field `thread_dim`") }; - (@munch [dyn_cache = $val:expr $(, $rest_f:ident = $rest_v:expr)*]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = (0); device = $device:tt; args = $a:tt) => { - $crate::offload!(@munch [$($rest_f = $rest_v),*]; kernel = $k; workgroup_dim = $w; thread_dim = $t; dyn_cache = (SOME $val); device = $device; args = $a) + (@munch [dyn_cache = $val:expr $(, $($rest:tt)*)?]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = (0); device = $device:tt; args = $a:tt) => { + $crate::offload!(@munch [ $($($rest)*)? ]; kernel = $k; workgroup_dim = $w; thread_dim = $t; dyn_cache = (SOME $val); device = $device; args = $a) }; - (@munch [dyn_cache = $val:expr $(, $rest_f:ident = $rest_v:expr)*]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = (SOME $old:expr); device = $device:tt; args = $a:tt) => { + (@munch [dyn_cache = $val:expr $(, $($rest:tt)*)?]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = (SOME $old:expr); device = $device:tt; args = $a:tt) => { compile_error!("duplicate field `dyn_cache`") }; - (@munch [device = $val:expr $(, $rest_f:ident = $rest_v:expr)*]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = NONE; args = $a:tt) => { - $crate::offload!(@munch [$($rest_f = $rest_v),*]; kernel = $k; workgroup_dim = $w; thread_dim = $t; dyn_cache = $d; device = (SOME $val); args = $a) + (@munch [device = $val:expr $(, $($rest:tt)*)?]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = NONE; args = $a:tt) => { + $crate::offload!(@munch [ $($($rest)*)? ]; kernel = $k; workgroup_dim = $w; thread_dim = $t; dyn_cache = $d; device = (SOME $val); args = $a) }; - (@munch [device = $val:expr $(, $rest_f:ident = $rest_v:expr)*]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = (SOME $old:expr); args = $a:tt) => { + (@munch [device = $val:expr $(, $($rest:tt)*)?]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = (SOME $old:expr); args = $a:tt) => { compile_error!("duplicate field `device`") }; - (@munch [args = $val:expr $(, $rest_f:ident = $rest_v:expr)*]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = NONE) => { - $crate::offload!(@munch [$($rest_f = $rest_v),*]; kernel = $k; workgroup_dim = $w; thread_dim = $t; dyn_cache = $d; device = $device; args = (SOME $val)) + (@munch [args = ($($arg:expr),* $(,)?) $(, $($rest:tt)*)?]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = NONE) => { + $crate::offload!(@munch [ $($($rest)*)? ]; kernel = $k; workgroup_dim = $w; thread_dim = $t; dyn_cache = $d; device = $device; args = (SOME_TUPLE ($($arg),*))) + }; + (@munch [args = $val:expr $(, $($rest:tt)*)?]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = NONE) => { + $crate::offload!(@munch [ $($($rest)*)? ]; kernel = $k; workgroup_dim = $w; thread_dim = $t; dyn_cache = $d; device = $device; args = (SOME $val)) }; - (@munch [args = $val:expr $(, $rest_f:ident = $rest_v:expr)*]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = (SOME $old:expr)) => { + (@munch [args = $val:expr $(, $($rest:tt)*)?]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = (SOME $old:expr)) => { + compile_error!("duplicate field `args`") + }; + (@munch [args = $val:expr $(, $($rest:tt)*)?]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = (SOME_TUPLE $old:tt)) => { compile_error!("duplicate field `args`") }; - (@munch [$invalid:ident = $val:expr $(, $rest_f:ident = $rest_v:expr)*]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = $a:tt) => { + (@munch [$invalid:ident = $val:expr $(, $($rest:tt)*)?]; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = $a:tt) => { compile_error!(concat!("unknown field `", stringify!($invalid), "`")) }; @@ -98,14 +94,32 @@ macro_rules! offload { (@munch []; kernel = $k:tt; workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = NONE) => { compile_error!("missing `args`") }; - (@munch []; kernel = (SOME $kernel:expr); workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = (SOME $args:expr)) => { + (@munch []; kernel = (SOME $kernel:expr); workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = (SOME_TUPLE ($($arg:expr),* $(,)?))) => {{ + #[allow(unused_imports)] + use $crate::offload::{DefaultLaunchCheck as _, RegionLaunchCheck as _}; + + let __offload_grid = $crate::offload!(@value $w); + let __offload_block = $crate::offload!(@value $t); + $crate::intrinsics::offload::<_, _, ()>( $kernel, - $crate::offload!(@value $w), - $crate::offload!(@value $t), + __offload_grid, + __offload_block, $crate::offload!(@value $d), $crate::offload!(@device $device), - $args, + ($({ + let __offload_arg = $arg; + (&__offload_arg).__offload_check_launch(__offload_grid, __offload_block); + __offload_arg + },)*), + ) + }}; + // Accepting an arbitrary expression here would let callers bypass the per-argument launch + // performed for `args = (..)` by binding the tuple to avariable first. + // FIXME(offload): run those checks on arbitrary expressions + (@munch []; kernel = (SOME $kernel:expr); workgroup_dim = $w:tt; thread_dim = $t:tt; dyn_cache = $d:tt; device = $device:tt; args = (SOME $args:expr)) => { + compile_error!( + "`args` must be a tuple literal" ) }; @@ -124,4 +138,201 @@ macro_rules! offload { ); device } }; + + ( $($tt:tt)* ) => { + $crate::offload!(@munch + [ $($tt)* ]; + kernel = NONE; + workgroup_dim = ([1, 1, 1]); + thread_dim = ([1, 1, 1]); + dyn_cache = (0); + device = NONE; + args = NONE + ) + }; +} + +// Region & Partitioning Strategy + +/// Error returned by [`PartitioningStrategy::check_launch`] when a launch +/// configuration is not compatible with a partitioning strategy. +#[derive(Debug)] +#[unstable(feature = "gpu_offload", issue = "131513")] +pub struct LaunchError { + message: &'static str, +} + +impl LaunchError { + /// Creates a new [`LaunchError`] with the given message. + #[unstable(feature = "gpu_offload", issue = "131513")] + pub const fn new(message: &'static str) -> Self { + Self { message } + } +} + +#[unstable(feature = "gpu_offload", issue = "131513")] +impl core::fmt::Display for LaunchError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(self.message) + } +} + +/// Defines how execution units access memory regions. +/// +/// # Safety +/// +/// Implementations must guarantee that generated views are disjoint. +/// +/// Implementations must also implement [`check_launch`] correctly since it is soundness-critical. +/// +/// [`check_launch`]: PartitioningStrategy::check_launch +#[unstable(feature = "gpu_offload", issue = "131513")] +pub unsafe trait PartitioningStrategy { + /// Read-only view type for the partitioned memory region. + type View<'a, T: 'a>; + + /// Mutable view type for the partitioned memory region. + type ViewMut<'a, T: 'a>; + + /// Returns the execution index of the current unit. + fn index() -> usize; + + /// Returns a read-only view of the region for the current execution context. + /// + /// # Safety + /// + /// `ptr` must point to `len` valid, initialized elements of type `T`. + /// The memory must stay valid for lifetime `'a`. + unsafe fn get<'a, T>(ptr: NonNull, len: usize) -> Option>; + + /// Returns a mutable view of the region for the current execution context. + /// + /// # Safety + /// + /// `ptr` must point to `len` valid, initialized elements of type `T`. + /// The memory must stay valid for lifetime `'a`. + /// The returned view must be disjoint from all other active views. + unsafe fn get_mut<'a, T>(ptr: NonNull, len: usize) -> Option>; + + /// Checks that a kernel using this strategy can be launched with `len` + /// elements on a `grid` by `block` launch configuration. + /// + /// This is called automatically before the kernel is launched. + /// + /// # Soundness + /// + /// This check is soundness-critical. It may only return `Ok` unconditionally + /// if and only if a launch is valid for arbitrary `len`, `grid` and `block` values. + /// If there is any configuration for which this strategy would produce incorrect + /// results or cause undefined behavior, it must return a [`LaunchError`] instead. + fn check_launch(len: usize, grid: [u32; 3], block: [u32; 3]) -> Result<(), LaunchError>; +} + +/// A memory region bound to a partitioning strategy. +#[derive(Debug)] +#[unstable(feature = "gpu_offload", issue = "131513")] +#[rustc_diagnostic_item = "offload_region"] +pub struct Region<'a, T, S: PartitioningStrategy> { + ptr: NonNull, + len: usize, + _marker: core::marker::PhantomData<(&'a mut [T], S)>, +} + +/// Raw representation used to build a [`Region`] from common aggregate types. +#[derive(Debug)] +#[unstable(feature = "gpu_offload", issue = "131513")] +pub struct RawRegion<'a, T> { + ptr: NonNull, + len: usize, + _marker: core::marker::PhantomData<&'a mut [T]>, +} + +impl<'a, T> From<&'a mut [T]> for RawRegion<'a, T> { + fn from(data: &'a mut [T]) -> Self { + // SAFETY: `data.as_mut_ptr()` is non-null, because it is derived from a reference. + let ptr = unsafe { NonNull::new_unchecked(data.as_mut_ptr()) }; + Self { ptr, len: data.len(), _marker: core::marker::PhantomData } + } +} + +impl<'a, T, const N: usize> From<&'a mut [T; N]> for RawRegion<'a, T> { + fn from(data: &'a mut [T; N]) -> Self { + // SAFETY: `data.as_mut_ptr()` is non-null, because it is derived from a reference. + let ptr = unsafe { NonNull::new_unchecked(data.as_mut_ptr()) }; + Self { ptr, len: N, _marker: core::marker::PhantomData } + } +} + +#[unstable(feature = "gpu_offload", issue = "131513")] +impl<'a, T, S: PartitioningStrategy> Region<'a, T, S> { + /// Creates a new partitioned region from data convertible into a [`RawRegion`]. + pub fn new(data: D) -> Self + where + D: Into>, + { + let raw = data.into(); + Self { ptr: raw.ptr, len: raw.len, _marker: core::marker::PhantomData } + } + + /// Returns a read-only view for the current execution context. + pub fn get(&self) -> Option> { + // SAFETY: `self.ptr` points to `self.len` valid elements for lifetime `'a`. + unsafe { S::get(self.ptr, self.len) } + } + + /// Returns a mutable view for the current execution context. + pub fn get_mut(&mut self) -> Option> { + // SAFETY: `self.ptr` points to `self.len` valid elements for lifetime `'a`. + // The strategy guarantees that the returned view is disjoint. + unsafe { S::get_mut(self.ptr, self.len) } + } + + /// Checks that this region can be launched with the given configuration. + /// + /// Called automatically by the [`offload!`] macro before launching a kernel. + pub fn check_launch(&self, grid: [u32; 3], block: [u32; 3]) -> Result<(), LaunchError> { + S::check_launch(self.len, grid, block) + } + + /// Reborrows the region, producing a new region that aliases the same memory with + /// the lifetime of the borrow. + pub fn reborrow(&mut self) -> Region<'_, T, S> { + Region { ptr: self.ptr, len: self.len, _marker: core::marker::PhantomData } + } +} + +// Launch validation helpers. + +/// Implementation detail of [`offload!`]: validates a [`Region`] argument against +/// the launch configuration of an offload call. +#[doc(hidden)] +#[unstable(feature = "gpu_offload", issue = "131513")] +pub trait RegionLaunchCheck { + /// Panics if the region's partitioning strategy rejects the launch. + fn __offload_check_launch(&self, grid: [u32; 3], block: [u32; 3]); +} + +#[unstable(feature = "gpu_offload", issue = "131513")] +impl<'a, T, S: PartitioningStrategy> RegionLaunchCheck for Region<'a, T, S> { + fn __offload_check_launch(&self, grid: [u32; 3], block: [u32; 3]) { + if let Err(err) = self.check_launch(grid, block) { + panic!( + "offload launch is not supported by the region's partitioning strategy: {}", + err, + ); + } + } +} + +/// no-op launch check for kernel arguments that are not [`Region`]s. +#[doc(hidden)] +#[unstable(feature = "gpu_offload", issue = "131513")] +pub trait DefaultLaunchCheck { + /// Does nothing. + fn __offload_check_launch(&self, grid: [u32; 3], block: [u32; 3]); +} + +#[unstable(feature = "gpu_offload", issue = "131513")] +impl DefaultLaunchCheck for &T { + fn __offload_check_launch(&self, _grid: [u32; 3], _block: [u32; 3]) {} } diff --git a/library/core/src/ops/bit.rs b/library/core/src/ops/bit.rs index 7c008531c1f5a..5b80b4edc20de 100644 --- a/library/core/src/ops/bit.rs +++ b/library/core/src/ops/bit.rs @@ -1,4 +1,5 @@ -/// The unary logical negation operator `!`. +/// The unary negation operator `!`. It performs either logical or bitwise negation, +/// depending on the type. /// /// # Examples /// @@ -37,15 +38,23 @@ pub const trait Not { #[stable(feature = "rust1", since = "1.0.0")] type Output; - /// Performs the unary `!` operation. + /// Performs the unary `!` operation using logical or bitwise negation. /// /// # Examples /// /// ``` + /// // Logical negation changes between true and false: /// assert_eq!(!true, false); /// assert_eq!(!false, true); + /// + /// // Bitwise negation changes between 0 and 1 in the binary + /// // representation of integers: /// assert_eq!(!1u8, 254); /// assert_eq!(!0u8, 255); + /// + /// // The same examples, represented in binary: + /// assert_eq!(!0b0000_0001u8, 0b1111_1110); + /// assert_eq!(!0b0000_0000u8, 0b1111_1111); /// ``` #[must_use] #[stable(feature = "rust1", since = "1.0.0")] diff --git a/library/core/src/ops/function.rs b/library/core/src/ops/function.rs index d40603e15bb1b..fd1bf57f75ce1 100644 --- a/library/core/src/ops/function.rs +++ b/library/core/src/ops/function.rs @@ -67,8 +67,8 @@ use crate::ptr::NonNull; // SAFETY: tidy is not smart enough to tell that the below unsafe block is a string label = "call the function in a closure: `|| unsafe {{ /* code */ }}`" ), - message = "expected an `{This:resolved}` closure, found `{Self}`", - label = "expected an `{This:resolved}` closure, found `{Self}`" + message = "expected `{this:resolved}` signature, found `{Self}`", + label = "expected `{this:resolved}` signature, found `{Self}`" )] #[fundamental] // so that regex can rely that `&str: !FnMut` #[must_use = "closures are lazy and do nothing unless called"] @@ -154,8 +154,8 @@ pub const trait Fn: [const] FnMut { // SAFETY: tidy is not smart enough to tell that the below unsafe block is a string label = "call the function in a closure: `|| unsafe {{ /* code */ }}`" ), - message = "expected an `{This:resolved}` closure, found `{Self}`", - label = "expected an `{This:resolved}` closure, found `{Self}`" + message = "expected `{this:resolved}` signature, found `{Self}`", + label = "expected `{this:resolved}` signature, found `{Self}`" )] #[fundamental] // so that regex can rely that `&str: !FnMut` #[must_use = "closures are lazy and do nothing unless called"] @@ -233,8 +233,8 @@ pub const trait FnMut: FnOnce { // SAFETY: tidy is not smart enough to tell that the below unsafe block is a string label = "call the function in a closure: `|| unsafe {{ /* code */ }}`" ), - message = "expected an `{This:resolved}` closure, found `{Self}`", - label = "expected an `{This:resolved}` closure, found `{Self}`" + message = "expected `{this:resolved}` signature, found `{Self}`", + label = "expected `{this:resolved}` signature, found `{Self}`" )] #[fundamental] // so that regex can rely that `&str: !FnMut` #[must_use = "closures are lazy and do nothing unless called"] diff --git a/library/core/src/ops/try_trait.rs b/library/core/src/ops/try_trait.rs index d0dd6c8d5c8b7..7fd273f2b1f50 100644 --- a/library/core/src/ops/try_trait.rs +++ b/library/core/src/ops/try_trait.rs @@ -118,12 +118,12 @@ use crate::ops::ControlFlow; on( all(from_desugaring = "TryBlock"), message = "a `try` block must return `Result` or `Option` \ - (or another type that implements `{This}`)", + (or another type that implements `{this}`)", label = "could not wrap the final value of the block as `{Self}` doesn't implement `Try`", ), on( all(from_desugaring = "QuestionMark"), - message = "the `?` operator can only be applied to values that implement `{This}`", + message = "the `?` operator can only be applied to values that implement `{this}`", label = "the `?` operator cannot be applied to type `{Self}`" ) )] @@ -232,7 +232,7 @@ pub const trait Try: [const] FromResidual { R = "core::option::Option", ), message = "the `?` operator can only be used on `Result`s, not `Option`s, \ - in {ItemContext} that returns `Result`", + in {item_context} that returns `Result`", label = "use `.ok_or(...)?` to provide an error compatible with `{Self}`", parent_label = "this function returns a `Result`" ), @@ -245,7 +245,7 @@ pub const trait Try: [const] FromResidual { // `From` in `?`, so this is not shown for result-in-result errors, // and thus it can be phrased more strongly than `ControlFlow`'s. message = "the `?` operator can only be used on `Result`s \ - in {ItemContext} that returns `Result`", + in {item_context} that returns `Result`", label = "this `?` produces `{R}`, which is incompatible with `{Self}`", parent_label = "this function returns a `Result`" ), @@ -256,7 +256,7 @@ pub const trait Try: [const] FromResidual { R = "core::result::Result", ), message = "the `?` operator can only be used on `Option`s, not `Result`s, \ - in {ItemContext} that returns `Option`", + in {item_context} that returns `Option`", label = "use `.ok()?` if you want to discard the `{R}` error information", parent_label = "this function returns an `Option`" ), @@ -268,7 +268,7 @@ pub const trait Try: [const] FromResidual { // `Option`-in-`Option` always works, as there's only one possible // residual, so this can also be phrased strongly. message = "the `?` operator can only be used on `Option`s \ - in {ItemContext} that returns `Option`", + in {item_context} that returns `Option`", label = "this `?` produces `{R}`, which is incompatible with `{Self}`", parent_label = "this function returns an `Option`" ), @@ -278,7 +278,7 @@ pub const trait Try: [const] FromResidual { Self = "core::ops::control_flow::ControlFlow", R = "core::ops::control_flow::ControlFlow", ), - message = "the `?` operator in {ItemContext} that returns `ControlFlow` \ + message = "the `?` operator in {item_context} that returns `ControlFlow` \ can only be used on other `ControlFlow`s (with the same Break type)", label = "this `?` produces `{R}`, which is incompatible with `{Self}`", parent_label = "this function returns a `ControlFlow`", @@ -291,16 +291,16 @@ pub const trait Try: [const] FromResidual { // `R` is not a `ControlFlow`, as that case was matched previously ), message = "the `?` operator can only be used on `ControlFlow`s \ - in {ItemContext} that returns `ControlFlow`", + in {item_context} that returns `ControlFlow`", label = "this `?` produces `{R}`, which is incompatible with `{Self}`", parent_label = "this function returns a `ControlFlow`", ), on( all(from_desugaring = "QuestionMark"), - message = "the `?` operator can only be used in {ItemContext} \ + message = "the `?` operator can only be used in {item_context} \ that returns `Result` or `Option` \ - (or another type that implements `{This}`)", - label = "cannot use the `?` operator in {ItemContext} that returns `{Self}`", + (or another type that implements `{this}`)", + label = "cannot use the `?` operator in {item_context} that returns `{Self}`", parent_label = "this function should return `Result` or `Option` to accept `?`" ), )] diff --git a/library/core/src/os/darwin/objc.rs b/library/core/src/os/darwin/objc.rs index 7be07891085a3..4512e3c3ada85 100644 --- a/library/core/src/os/darwin/objc.rs +++ b/library/core/src/os/darwin/objc.rs @@ -74,7 +74,7 @@ pub type SEL = *mut objc_selector; /// /// let string_class = objc::class!("NSString"); /// ``` -#[allow_internal_unstable(rustc_attrs)] +#[allow_internal_unstable(core_os, rustc_attrs)] pub macro class($classname:expr) {{ // Since static Objective-C class references actually end up with multiple definitions // across dylib boundaries, we only expose the value of the static and don't provide a way to @@ -102,7 +102,7 @@ pub macro class($classname:expr) {{ /// let alloc_sel = objc::selector!("alloc"); /// let init_sel = objc::selector!("initWithCString:encoding:"); /// ``` -#[allow_internal_unstable(rustc_attrs)] +#[allow_internal_unstable(core_os, rustc_attrs)] pub macro selector($methname:expr) {{ // Since static Objective-C selector references actually end up with multiple definitions // across dylib boundaries, we only expose the value of the static and don't provide a way to diff --git a/library/core/src/os/mod.rs b/library/core/src/os/mod.rs index 8d177dd3947c9..ba86ba7cd0fb5 100644 --- a/library/core/src/os/mod.rs +++ b/library/core/src/os/mod.rs @@ -1,6 +1,5 @@ //! OS-specific functionality. - -#![unstable(feature = "darwin_objc", issue = "145496")] +#![unstable(feature = "core_os", issue = "none")] #![allow(missing_docs)] #[cfg(all( @@ -23,3 +22,6 @@ pub mod darwin {} )))] #[cfg(any(target_vendor = "apple", doc))] pub mod darwin; + +#[cfg(any(target_family = "windows", doc))] +pub mod windows; diff --git a/library/core/src/os/windows.rs b/library/core/src/os/windows.rs new file mode 100644 index 0000000000000..47528f67f6428 --- /dev/null +++ b/library/core/src/os/windows.rs @@ -0,0 +1,39 @@ +//! Platform-specific extensions to `core` for Windows platforms. + +#![unstable(feature = "codeview_annotation", issue = "163964")] +#![doc(cfg(windows))] + +#[doc(inline)] +pub use crate::intrinsics::CodeViewAnnotationArgs; + +/// Writes [`T::ARGS`](CodeViewAnnotationArgs::ARGS) to the PDB as an `S_ANNOTATION` record using +/// [`llvm.codeview.annotation`](https://llvm.org/docs/LangRef.html#llvm-codeview-annotation-intrinsic). +/// +/// This function has an effect only on the `msvc` environment and the LLVM backend. It is a no-op on other +/// environments and backends. +/// +/// # Examples +/// +/// To call `codeview_annotation`, the caller must declare a type +/// implementing [`CodeViewAnnotationArgs`] and pass it as the type +/// parameter to `codeview_annotation`. The string arguments must be +/// specified in `CodeViewAnnotationArgs::ARGS`. +/// +#[cfg_attr(windows, doc = "```")] +#[cfg_attr(not(windows), doc = "```ignore (needs windows)")] +/// #![feature(codeview_annotation, core_os)] +/// use core::os::windows::{codeview_annotation, CodeViewAnnotationArgs}; +/// +/// struct Args; +/// +/// impl CodeViewAnnotationArgs for Args { +/// const ARGS: &[&str] = &["Hello", "World"]; +/// } +/// +/// codeview_annotation::(); +/// ``` +#[inline(always)] +#[unstable(feature = "codeview_annotation", issue = "163964")] +pub fn codeview_annotation() { + crate::intrinsics::codeview_annotation::(); +} diff --git a/library/core/src/panic.rs b/library/core/src/panic.rs index 4fcd5f615e993..52fcfebd1111c 100644 --- a/library/core/src/panic.rs +++ b/library/core/src/panic.rs @@ -97,8 +97,9 @@ pub macro unreachable_2021 { /// Invokes a closure, aborting if the closure unwinds. /// /// When compiled with aborting panics, this function is effectively a no-op. -/// With unwinding panics, an unwind results in another call into the panic -/// hook followed by a process abort. +/// With unwinding panics, an unwind that would leave the closure results in another call into the +/// panic handler / panic hook followed by a process abort. (Locals in the closure get dropped as +/// usual.) Unwinds that get caught inside the closure still behave as normal. /// /// # Notes /// @@ -114,7 +115,7 @@ pub macro unreachable_2021 { /// to prevent unwinds. However, note that `extern "C" fn` will automatically /// convert unwinds to aborts, so using this function isn't necessary for FFI. #[unstable(feature = "abort_unwind", issue = "130338")] -#[rustc_nounwind] +#[rustc_nounwind] // this attribute does all the actual work here pub fn abort_on_unwind R, R>(f: F) -> R { f() } diff --git a/library/core/src/panic/unwind_safe.rs b/library/core/src/panic/unwind_safe.rs index 60c833db33e15..79141ec4370d7 100644 --- a/library/core/src/panic/unwind_safe.rs +++ b/library/core/src/panic/unwind_safe.rs @@ -5,7 +5,7 @@ use crate::future::Future; use crate::marker::PointeeSized; use crate::ops::{Deref, DerefMut}; use crate::pin::Pin; -use crate::ptr::{NonNull, Unique}; +use crate::ptr::NonNull; use crate::task::{Context, Poll}; /// A marker trait which represents "panic safe" types in Rust. @@ -186,8 +186,6 @@ impl UnwindSafe for &T {} impl UnwindSafe for *const T {} #[stable(feature = "catch_unwind", since = "1.9.0")] impl UnwindSafe for *mut T {} -#[unstable(feature = "ptr_internals", issue = "none")] -impl UnwindSafe for Unique {} #[stable(feature = "nonnull", since = "1.25.0")] impl UnwindSafe for NonNull {} #[stable(feature = "catch_unwind", since = "1.9.0")] diff --git a/library/core/src/process.rs b/library/core/src/process.rs index 1c2769f07895b..d03e7cba04bef 100644 --- a/library/core/src/process.rs +++ b/library/core/src/process.rs @@ -22,7 +22,7 @@ /// /// `abort_immediate` lowers to a trap instruction on *most* architectures; on /// some architectures it simply lowers to call the unmangled `abort` function. -/// The exact behavior is architecture and system dependent. +/// The exact behavior is architecture- and system-dependent. /// /// On bare-metal (no OS) systems the trap instruction usually causes a /// *hardware* exception to be raised in a *synchronous* fashion; hardware @@ -34,11 +34,71 @@ /// corresponds to `SIGILL` or equivalent, *unless* this signal is handled. /// Other signals such as `SIGABRT`, `SIGTRAP`, `SIGSEGV`, and `SIGBUS` may be /// produced instead, depending on specifics. This is not an exhaustive list. +/// +/// # Examples +/// +/// ```no_run +/// #![feature(abort_immediate)] +/// +/// use core::process; +/// +/// fn main() { +/// println!("aborting"); +/// +/// process::abort_immediate(); +/// +/// // execution never gets here +/// } +/// ``` +/// +/// The `abort_immediate` function terminates the process, so the destructor +/// will not get run on the example below: +/// +/// ```no_run +/// #![feature(abort_immediate)] +/// +/// use core::process; +/// +/// struct HasDrop; +/// +/// impl Drop for HasDrop { +/// fn drop(&mut self) { +/// println!("This will never be printed!"); +/// } +/// } +/// +/// fn main() { +/// let _x = HasDrop; +/// process::abort_immediate(); +/// // the destructor implemented for HasDrop will never get run +/// } +/// ``` +/// #[unstable(feature = "abort_immediate", issue = "154601")] #[rustc_const_unstable(feature = "abort_immediate", issue = "154601")] #[cold] #[cfg_attr(miri, track_caller)] // even without panics, this helps for Miri backtraces #[doc(alias = "halt")] +#[doc(alias = "trap")] pub const fn abort_immediate() -> ! { crate::intrinsics::abort_immediate() } + +/// Triggers a debugger breakpoint, pausing execution if possible. +/// +/// This signals the environment to pause execution if a debugger is attached, and pass control to +/// the debugger. Execution may then be aborted or continued by the user controlling the debugger. +/// +/// If possible, this will produce an instruction sequence that allows a debugger to resume *after* +/// the breakpoint, rather than resuming *at* the breakpoint; however, the exact behavior is +/// target-specific and debugger-specific, and not guaranteed. +/// +/// If no debugger is attached, this will typically abort the process but may also continue +/// execution. +// NOTE: this is meant to discuss alternatives to `arch::breakpoint`. We may or may not +// actually want to stabilize both of them. +#[unstable(feature = "breakpoint", issue = "133724")] +#[inline(always)] +pub fn resumable_breakpoint() { + crate::intrinsics::breakpoint(); +} diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs index 7e568140fa10d..4121d40b75629 100644 --- a/library/core/src/ptr/mod.rs +++ b/library/core/src/ptr/mod.rs @@ -449,10 +449,6 @@ mod non_null; #[stable(feature = "nonnull", since = "1.25.0")] pub use non_null::NonNull; -mod unique; -#[unstable(feature = "ptr_internals", issue = "none")] -pub use unique::Unique; - mod const_ptr; mod mut_ptr; @@ -2599,47 +2595,46 @@ pub fn addr_eq(p: *const T, q: *const U) -> bo /// This is the same as `f == g`, but using this function makes clear that the potentially /// surprising semantics of function pointer comparison are involved. /// -/// There are **very few guarantees** about how functions are compiled and they have no intrinsic +/// There are **basically no guarantees** about how functions are compiled as they have no intrinsic /// “identity”; in particular, this comparison: /// -/// * May return `true` unexpectedly, in cases where functions are equivalent. +/// * May return `true` unexpectedly, even in cases where functions are not logically equivalent, +/// as long as they compile to the same machine code. /// /// For example, the following program is likely (but not guaranteed) to print `(true, true)` /// when compiled with optimization: /// /// ``` +/// use std::hint::assert_unchecked; +/// /// let f: fn(i32) -> i32 = |x| x; -/// let g: fn(i32) -> i32 = |x| x + 0; // different closure, different body -/// let h: fn(u32) -> u32 = |x| x + 0; // different signature too -/// dbg!(std::ptr::fn_addr_eq(f, g), std::ptr::fn_addr_eq(f, h)); // not guaranteed to be equal +/// let g: fn(u32) -> u32 = |x| x + 0; // different type, different body +/// let h: unsafe fn(i32) -> i32 = |x| unsafe { assert_unchecked(x != 0); x }; // different unsafe preconditions +/// // These three functions are allowed but not guaranteed to compare equal. +/// dbg!(std::ptr::fn_addr_eq(f, g), std::ptr::fn_addr_eq(f, h)); /// ``` /// +/// In other words, functions that compare equal can have different behavior! Even if +/// `fn_addr_eq(f, h)` is true, it is still undefined behavior to call `h(0)`, while `f(0)` is +/// safe to call. +/// +/// The explanation for this is that function pointers have [provenance][crate::ptr#provenance]. +/// If functions have the same assembly and get merged, their provenance decides which variant of +/// the function will be called. `f` and `h` therefore end up with the same address but different +/// provenance, and `fn_addr_eq` only compares the address. +/// /// * May return `false` in any case. /// /// This is particularly likely with generic functions but may happen with any function. /// (From an implementation perspective, this is possible because functions may sometimes be /// processed more than once by the compiler, resulting in duplicate machine code.) /// -/// Despite these false positives and false negatives, this comparison can still be useful. -/// Specifically, if -/// -/// * `T` is the same type as `U`, `T` is a [subtype] of `U`, or `U` is a [subtype] of `T`, and -/// * `ptr::fn_addr_eq(f, g)` returns true, -/// -/// then calling `f` and calling `g` will be equivalent. -/// -/// -/// # Examples -/// -/// ``` -/// use std::ptr; -/// -/// fn a() { println!("a"); } -/// fn b() { println!("b"); } -/// assert!(!ptr::fn_addr_eq(a as fn(), b as fn())); -/// ``` -/// -/// [subtype]: https://doc.rust-lang.org/reference/subtyping.html +/// These limitations imply that comparing function pointers (via this function or via `==`) is only +/// useful in extremely niche circumstances. For instance, if a library maintains a set of +/// callbacks, and doesn't guarantee which of them are actually called (i.e., callbacks can be +/// spuriously skipped), then it can be legitimate to deduplicate callbacks based on `fn_addr_eq`. +/// This can never introduce undefined behavior because it only skips executing some of the +/// registered callbacks. #[stable(feature = "ptr_fn_addr_eq", since = "1.85.0")] #[inline(always)] #[must_use = "function pointer comparison produces a value"] diff --git a/library/core/src/ptr/non_null.rs b/library/core/src/ptr/non_null.rs index 52c4dd557f404..3d826c3c5a3ac 100644 --- a/library/core/src/ptr/non_null.rs +++ b/library/core/src/ptr/non_null.rs @@ -8,7 +8,6 @@ use crate::marker::{Destruct, PointeeSized, Unsize}; use crate::mem::{MaybeUninit, SizedTypeProperties, transmute}; use crate::num::NonZero; use crate::ops::{CoerceUnsized, DispatchFromDyn}; -use crate::ptr::Unique; use crate::slice::{self, SliceIndex}; #[cfg(kani)] use crate::ub_checks; @@ -84,7 +83,7 @@ use crate::{fmt, hash, intrinsics, mem, ptr}; #[lang = "non_null"] // Flux cannot lower the pattern-type field; treat the struct as opaque. #[cfg_attr(flux, flux::opaque)] -pub struct NonNull { +pub struct NonNull<#[rustc_assert_variance(covariant)] T: PointeeSized> { pointer: crate::pattern_type!(*const T is !null), } @@ -1934,15 +1933,6 @@ impl hash::Hash for NonNull { } } -#[unstable(feature = "ptr_internals", issue = "none")] -#[rustc_const_unstable(feature = "const_convert", issue = "143773")] -const impl From> for NonNull { - #[inline] - fn from(unique: Unique) -> Self { - unique.as_non_null_ptr() - } -} - #[stable(feature = "nonnull", since = "1.25.0")] #[rustc_const_unstable(feature = "const_convert", issue = "143773")] const impl From<&mut T> for NonNull { diff --git a/library/core/src/result.rs b/library/core/src/result.rs index 9d898df3c6d3c..255c93bd8109c 100644 --- a/library/core/src/result.rs +++ b/library/core/src/result.rs @@ -1181,7 +1181,7 @@ impl Result { { match self { Ok(t) => t, - Err(e) => unwrap_failed(msg, &e), + Err(e) => unwrap_failed(msg, e), } } @@ -1229,7 +1229,7 @@ impl Result { { match self { Ok(t) => t, - Err(e) => unwrap_failed("called `Result::unwrap()` on an `Err` value", &e), + Err(e) => unwrap_failed("called `Result::unwrap()` on an `Err` value", e), } } @@ -1294,7 +1294,7 @@ impl Result { T: fmt::Debug, { match self { - Ok(t) => unwrap_failed(msg, &t), + Ok(t) => unwrap_failed(msg, t), Err(e) => e, } } @@ -1325,7 +1325,7 @@ impl Result { T: fmt::Debug, { match self { - Ok(t) => unwrap_failed("called `Result::unwrap_err()` on an `Ok` value", &t), + Ok(t) => unwrap_failed("called `Result::unwrap_err()` on an `Ok` value", t), Err(e) => e, } } @@ -1858,12 +1858,22 @@ impl Result, E> { } // This is a separate function to reduce the code size of the methods +// +// Take `error` by value so that `unwrap` doesn't need to include a landing pad to drop `error` +// during unwinding. This reduces the size of `unwrap` and allows it to be MIR inlined. #[cfg(not(panic = "immediate-abort"))] -#[inline(never)] +#[rustc_no_mir_inline] #[cold] #[track_caller] -fn unwrap_failed(msg: &str, error: &dyn fmt::Debug) -> ! { - panic!("{msg}: {error:?}"); +fn unwrap_failed(msg: &str, error: T) -> ! { + // Do the actual formatting with a `&dyn Debug` to reduce code size. + #[inline(never)] + #[cold] + #[track_caller] + fn unwrap_failed_inner(msg: &str, error: &dyn fmt::Debug) -> ! { + panic!("{msg}: {error:?}") + } + unwrap_failed_inner(msg, &error) } // This is a separate function to avoid constructing a `dyn Debug` @@ -1874,7 +1884,7 @@ fn unwrap_failed(msg: &str, error: &dyn fmt::Debug) -> ! { #[inline] #[cold] #[track_caller] -const fn unwrap_failed(_msg: &str, _error: &T) -> ! { +const fn unwrap_failed(_msg: &str, _error: T) -> ! { panic!() } diff --git a/library/core/src/slice/cmp.rs b/library/core/src/slice/cmp.rs index fcdde758fa7df..3abf75cbf5e2b 100644 --- a/library/core/src/slice/cmp.rs +++ b/library/core/src/slice/cmp.rs @@ -70,35 +70,27 @@ const impl PartialOrd for [T] { // see , or we generate // niche discriminant checks in a way that doesn't trigger it. - as_underlying(self.__chaining_lt(other)) == 1 + as_underlying(self.__chaining_less(other)) == 1 } #[inline] fn le(&self, other: &Self) -> bool { - as_underlying(self.__chaining_le(other)) != 0 + as_underlying(self.__chaining_less(other)) != 0 } #[inline] fn gt(&self, other: &Self) -> bool { - as_underlying(self.__chaining_gt(other)) == 1 + as_underlying(self.__chaining_greater(other)) == 1 } #[inline] fn ge(&self, other: &Self) -> bool { - as_underlying(self.__chaining_ge(other)) != 0 + as_underlying(self.__chaining_greater(other)) != 0 } #[inline] - fn __chaining_lt(&self, other: &Self) -> ControlFlow { - SliceChain::chaining_lt(self, other) + fn __chaining_less(&self, other: &Self) -> ControlFlow { + SliceChain::chaining_less(self, other) } #[inline] - fn __chaining_le(&self, other: &Self) -> ControlFlow { - SliceChain::chaining_le(self, other) - } - #[inline] - fn __chaining_gt(&self, other: &Self) -> ControlFlow { - SliceChain::chaining_gt(self, other) - } - #[inline] - fn __chaining_ge(&self, other: &Self) -> ControlFlow { - SliceChain::chaining_ge(self, other) + fn __chaining_greater(&self, other: &Self) -> ControlFlow { + SliceChain::chaining_greater(self, other) } } @@ -167,10 +159,8 @@ const trait SlicePartialOrd: Sized { #[rustc_const_unstable(feature = "const_cmp", issue = "143800")] // intermediate trait for specialization of slice's PartialOrd chaining methods const trait SliceChain: Sized { - fn chaining_lt(left: &[Self], right: &[Self]) -> ControlFlow; - fn chaining_le(left: &[Self], right: &[Self]) -> ControlFlow; - fn chaining_gt(left: &[Self], right: &[Self]) -> ControlFlow; - fn chaining_ge(left: &[Self], right: &[Self]) -> ControlFlow; + fn chaining_less(left: &[Self], right: &[Self]) -> ControlFlow; + fn chaining_greater(left: &[Self], right: &[Self]) -> ControlFlow; } type AlwaysBreak = ControlFlow; @@ -192,17 +182,11 @@ const impl SlicePartialOrd for A { #[rustc_const_unstable(feature = "const_cmp", issue = "143800")] const impl SliceChain for A { - default fn chaining_lt(left: &[Self], right: &[Self]) -> ControlFlow { - chaining_impl(left, right, PartialOrd::__chaining_lt, usize::__chaining_lt) + default fn chaining_less(left: &[Self], right: &[Self]) -> ControlFlow { + chaining_impl(left, right, PartialOrd::__chaining_less, usize::__chaining_less) } - default fn chaining_le(left: &[Self], right: &[Self]) -> ControlFlow { - chaining_impl(left, right, PartialOrd::__chaining_le, usize::__chaining_le) - } - default fn chaining_gt(left: &[Self], right: &[Self]) -> ControlFlow { - chaining_impl(left, right, PartialOrd::__chaining_gt, usize::__chaining_gt) - } - default fn chaining_ge(left: &[Self], right: &[Self]) -> ControlFlow { - chaining_impl(left, right, PartialOrd::__chaining_ge, usize::__chaining_ge) + default fn chaining_greater(left: &[Self], right: &[Self]) -> ControlFlow { + chaining_impl(left, right, PartialOrd::__chaining_greater, usize::__chaining_greater) } } @@ -343,33 +327,19 @@ const impl SliceOrd for A { #[rustc_const_unstable(feature = "const_cmp", issue = "143800")] const impl SliceChain for A { #[inline] - fn chaining_lt(left: &[Self], right: &[Self]) -> ControlFlow { + fn chaining_less(left: &[Self], right: &[Self]) -> ControlFlow { match SliceOrd::compare(left, right) { Ordering::Equal => ControlFlow::Continue(()), ne => ControlFlow::Break(ne.is_lt()), } } #[inline] - fn chaining_le(left: &[Self], right: &[Self]) -> ControlFlow { - match SliceOrd::compare(left, right) { - Ordering::Equal => ControlFlow::Continue(()), - ne => ControlFlow::Break(ne.is_le()), - } - } - #[inline] - fn chaining_gt(left: &[Self], right: &[Self]) -> ControlFlow { + fn chaining_greater(left: &[Self], right: &[Self]) -> ControlFlow { match SliceOrd::compare(left, right) { Ordering::Equal => ControlFlow::Continue(()), ne => ControlFlow::Break(ne.is_gt()), } } - #[inline] - fn chaining_ge(left: &[Self], right: &[Self]) -> ControlFlow { - match SliceOrd::compare(left, right) { - Ordering::Equal => ControlFlow::Continue(()), - ne => ControlFlow::Break(ne.is_ge()), - } - } } pub(super) trait SliceContains: Sized { diff --git a/library/core/src/slice/mod.rs b/library/core/src/slice/mod.rs index 6b50de1455918..a65ecd0bb1c61 100644 --- a/library/core/src/slice/mod.rs +++ b/library/core/src/slice/mod.rs @@ -2565,7 +2565,6 @@ impl [T] { /// # Examples /// /// ``` - /// #![feature(slice_split_once)] /// let s = [1, 2, 3, 2, 4]; /// assert_eq!(s.split_once(|&x| x == 2), Some(( /// &[1][..], @@ -2573,7 +2572,7 @@ impl [T] { /// ))); /// assert_eq!(s.split_once(|&x| x == 0), None); /// ``` - #[unstable(feature = "slice_split_once", issue = "112811")] + #[stable(feature = "slice_split_once", since = "CURRENT_RUSTC_VERSION")] #[inline] pub fn split_once(&self, pred: F) -> Option<(&[T], &[T])> where @@ -2594,7 +2593,6 @@ impl [T] { /// # Examples /// /// ``` - /// #![feature(slice_split_once)] /// let s = [1, 2, 3, 2, 4]; /// assert_eq!(s.rsplit_once(|&x| x == 2), Some(( /// &[1, 2, 3][..], @@ -2602,7 +2600,7 @@ impl [T] { /// ))); /// assert_eq!(s.rsplit_once(|&x| x == 0), None); /// ``` - #[unstable(feature = "slice_split_once", issue = "112811")] + #[stable(feature = "slice_split_once", since = "CURRENT_RUSTC_VERSION")] #[inline] pub fn rsplit_once(&self, pred: F) -> Option<(&[T], &[T])> where diff --git a/library/core/src/task/wake.rs b/library/core/src/task/wake.rs index 1dbd11426e57e..635917ea6973b 100644 --- a/library/core/src/task/wake.rs +++ b/library/core/src/task/wake.rs @@ -843,9 +843,10 @@ impl LocalWaker { /// Creates a new `LocalWaker` from [`RawWaker`]. /// + /// # Safety + /// /// The behavior of the returned `LocalWaker` is undefined if the contract defined /// in [`RawWaker`]'s and [`RawWakerVTable`]'s documentation is not upheld. - /// Therefore this method is unsafe. #[inline] #[must_use] #[unstable(feature = "local_waker", issue = "118959")] diff --git a/library/core/src/tuple.rs b/library/core/src/tuple.rs index 6dd26d3ed762d..cb61ffadee908 100644 --- a/library/core/src/tuple.rs +++ b/library/core/src/tuple.rs @@ -71,35 +71,27 @@ macro_rules! tuple_impls { } #[inline] fn lt(&self, other: &($($T,)+)) -> bool { - lexical_ord!(lt, __chaining_lt, $( ${ignore($T)} self.${index()}, other.${index()} ),+) + lexical_ord!(lt, __chaining_less, $( ${ignore($T)} self.${index()}, other.${index()} ),+) } #[inline] fn le(&self, other: &($($T,)+)) -> bool { - lexical_ord!(le, __chaining_le, $( ${ignore($T)} self.${index()}, other.${index()} ),+) + lexical_ord!(le, __chaining_less, $( ${ignore($T)} self.${index()}, other.${index()} ),+) } #[inline] fn ge(&self, other: &($($T,)+)) -> bool { - lexical_ord!(ge, __chaining_ge, $( ${ignore($T)} self.${index()}, other.${index()} ),+) + lexical_ord!(ge, __chaining_greater, $( ${ignore($T)} self.${index()}, other.${index()} ),+) } #[inline] fn gt(&self, other: &($($T,)+)) -> bool { - lexical_ord!(gt, __chaining_gt, $( ${ignore($T)} self.${index()}, other.${index()} ),+) + lexical_ord!(gt, __chaining_greater, $( ${ignore($T)} self.${index()}, other.${index()} ),+) } #[inline] - fn __chaining_lt(&self, other: &($($T,)+)) -> ControlFlow { - lexical_chain!(__chaining_lt, $( ${ignore($T)} self.${index()}, other.${index()} ),+) + fn __chaining_less(&self, other: &($($T,)+)) -> ControlFlow { + lexical_chain!(__chaining_less, $( ${ignore($T)} self.${index()}, other.${index()} ),+) } #[inline] - fn __chaining_le(&self, other: &($($T,)+)) -> ControlFlow { - lexical_chain!(__chaining_le, $( ${ignore($T)} self.${index()}, other.${index()} ),+) - } - #[inline] - fn __chaining_gt(&self, other: &($($T,)+)) -> ControlFlow { - lexical_chain!(__chaining_gt, $( ${ignore($T)} self.${index()}, other.${index()} ),+) - } - #[inline] - fn __chaining_ge(&self, other: &($($T,)+)) -> ControlFlow { - lexical_chain!(__chaining_ge, $( ${ignore($T)} self.${index()}, other.${index()} ),+) + fn __chaining_greater(&self, other: &($($T,)+)) -> ControlFlow { + lexical_chain!(__chaining_greater, $( ${ignore($T)} self.${index()}, other.${index()} ),+) } } } @@ -206,13 +198,14 @@ macro_rules! lexical_ord { // Same parameter interleaving as `lexical_ord` above macro_rules! lexical_chain { - ($chain_rel: ident, $a:expr, $b:expr $(,$rest_a:expr, $rest_b:expr)*) => {{ - PartialOrd::$chain_rel(&$a, &$b)?; - lexical_chain!($chain_rel $(,$rest_a, $rest_b)*) - }}; - ($chain_rel: ident) => { + ($chain_rel:ident $(, $a:expr, $b:expr)* $(,)?) => {{ + // Using `?` here makes this harder on mir-opt due + // to the `Try` residuals bloating the function + $(if let r @ Break(_) = PartialOrd::$chain_rel(&$a, &$b) { + return r; + })* Continue(()) - }; + }}; } macro_rules! lexical_partial_cmp { diff --git a/library/coretests/tests/bstr.rs b/library/coretests/tests/byte_str.rs similarity index 99% rename from library/coretests/tests/bstr.rs rename to library/coretests/tests/byte_str.rs index dffc7e3f03494..bc90b348446bf 100644 --- a/library/coretests/tests/bstr.rs +++ b/library/coretests/tests/byte_str.rs @@ -1,4 +1,4 @@ -use core::bstr::ByteStr; +use core::byte_str::ByteStr; #[test] fn test_debug() { diff --git a/library/coretests/tests/iter/adapters/skip.rs b/library/coretests/tests/iter/adapters/skip.rs index 8d5d06ad9fb3a..cfd00ac609715 100644 --- a/library/coretests/tests/iter/adapters/skip.rs +++ b/library/coretests/tests/iter/adapters/skip.rs @@ -86,6 +86,31 @@ fn test_skip_advance_by() { assert_eq!((0..0).skip(1).advance_back_by(0), Ok(())); } +#[test] +fn test_skip_unfused_advance_by() { + /// A legal non-fused iterator: `None` once, then `Some(1)`, `Some(2)`, ... + struct Gap(u32); + + impl Iterator for Gap { + type Item = u32; + fn next(&mut self) -> Option { + self.0 += 1; + if self.0 == 1 { None } else { Some(self.0 - 1) } + } + } + + assert_eq!(Gap(0).skip(1).next(), None); + + let mut iter = Gap(0).skip(1); + assert_eq!(iter.advance_by(0), Ok(())); + // `Ok` means we didn't see the (unfused) end yet... + assert_eq!(iter.next(), None); + + // Chain also depends on this behavior internally + assert_eq!(Gap(0).skip(1).chain(empty()).next(), None); + assert_eq!(Gap(0).skip(1).chain(empty()).nth(0), None); +} + #[test] fn test_iterator_skip_count() { let xs = [0, 1, 2, 3, 5, 13, 15, 16, 17, 19, 20, 30]; diff --git a/library/coretests/tests/lib.rs b/library/coretests/tests/lib.rs index 1fdb0c820d662..8f2fd10b9e475 100644 --- a/library/coretests/tests/lib.rs +++ b/library/coretests/tests/lib.rs @@ -10,7 +10,7 @@ #![feature(async_iterator)] #![feature(atomic_volatile)] #![feature(borrowed_buf_init)] -#![feature(bstr)] +#![feature(byte_str)] #![feature(casefold)] #![feature(cfg_overflow_checks)] #![feature(cfg_target_has_reliable_f16_f128)] @@ -107,7 +107,6 @@ #![feature(slice_internals)] #![feature(slice_partition_dedup)] #![feature(slice_shift)] -#![feature(slice_split_once)] #![feature(sliceindex_wrappers)] #![feature(splat)] #![feature(split_array)] @@ -179,7 +178,7 @@ mod asserting; mod async_iter; mod atomic; mod bool; -mod bstr; +mod byte_str; mod cell; mod char; mod clone; diff --git a/library/coretests/tests/net/ip_addr.rs b/library/coretests/tests/net/ip_addr.rs index ea78d7c58e9e4..67517e025b060 100644 --- a/library/coretests/tests/net/ip_addr.rs +++ b/library/coretests/tests/net/ip_addr.rs @@ -463,6 +463,7 @@ fn ipv4_properties() { check!("192.0.0.11"); check!("192.0.0.255"); check!("192.0.0.100"); + check!("192.88.99.2"); check!("240.0.0.0", reserved); check!("251.54.1.76", reserved); check!("254.255.255.255", reserved); @@ -632,6 +633,7 @@ fn ipv6_properties() { ); check!("100::", &[0x01, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], unicast_global); + check!("100:0:0:1::", &[1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0], unicast_global); check!("2001::", &[0x20, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], unicast_global); @@ -647,6 +649,12 @@ fn ipv6_properties() { global | unicast_global ); + check!( + "2001:1::3", + &[0x20, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3], + global | unicast_global + ); + check!( "2001:3::", &[0x20, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], diff --git a/library/std/Cargo.toml b/library/std/Cargo.toml index 81e6fd3ad6218..b7cc848a10c4e 100644 --- a/library/std/Cargo.toml +++ b/library/std/Cargo.toml @@ -34,7 +34,7 @@ miniz_oxide = { version = "0.9.0", optional = true, default-features = false } addr2line = { version = "0.27.0", optional = true, default-features = false } [target.'cfg(not(all(windows, target_env = "msvc")))'.dependencies] -libc = { version = "0.2.189", default-features = false, features = [ +libc = { version = "0.2.191", default-features = false, features = [ 'rustc-dep-of-std', ], public = true } diff --git a/library/std/src/bstr.rs b/library/std/src/bstr.rs deleted file mode 100644 index dd49177162833..0000000000000 --- a/library/std/src/bstr.rs +++ /dev/null @@ -1,4 +0,0 @@ -//! The `ByteStr` and `ByteString` types and trait implementations. - -#[unstable(feature = "bstr", issue = "134915")] -pub use alloc::bstr::{ByteStr, ByteString}; diff --git a/library/std/src/byte_str.rs b/library/std/src/byte_str.rs new file mode 100644 index 0000000000000..48703b52ece21 --- /dev/null +++ b/library/std/src/byte_str.rs @@ -0,0 +1,4 @@ +//! The `ByteStr` and `ByteString` types and trait implementations. + +#[unstable(feature = "byte_str", issue = "134915")] +pub use alloc::byte_str::{ByteStr, ByteString}; diff --git a/library/std/src/fs.rs b/library/std/src/fs.rs index 139275ed2e1c2..bf61ab79964c4 100644 --- a/library/std/src/fs.rs +++ b/library/std/src/fs.rs @@ -2257,10 +2257,9 @@ impl Metadata { FileType(self.0.file_type()) } - /// Returns `true` if this metadata is for a directory. The - /// result is mutually exclusive to the result of - /// [`Metadata::is_file`], and will be false for symlink metadata - /// obtained from [`symlink_metadata`]. + /// Returns `true` if this metadata is for a directory. The result is mutually exclusive to the + /// result of [`Metadata::is_file`] and [`Metadata::is_symlink`] (i.e., at most one of these + /// three methods returns `true`). /// /// # Examples /// @@ -2280,10 +2279,9 @@ impl Metadata { self.file_type().is_dir() } - /// Returns `true` if this metadata is for a regular file. The - /// result is mutually exclusive to the result of - /// [`Metadata::is_dir`], and will be false for symlink metadata - /// obtained from [`symlink_metadata`]. + /// Returns `true` if this metadata is for a regular file. The result is mutually exclusive to + /// the result of [`Metadata::is_dir`] and [`Metadata::is_symlink`] (i.e., at most one of these + /// three methods returns `true`). /// /// When the goal is simply to read from (or write to) the source, the most /// reliable way to test the source can be read (or written to) is to open @@ -2797,6 +2795,20 @@ impl AsInner for Permissions { } } +impl ReadDir { + /// Returns a handle for the directory this `ReadDir` is reading. + /// + /// Whenever possible, the new handle will be created in a race-free manner from the underlying + /// handle used by `ReadDir`. However, this may fall back to opening a new handle based on the + /// path, which is subject to race conditions if the file system has changed since the `ReadDir` + /// was created. + // FIXME: before stabilization, ensure that we have a race-free implementation on Windows! + #[unstable(feature = "dirfd", issue = "120426")] + pub fn dir(&self) -> io::Result { + self.0.dir().map(|d| Dir { inner: d }) + } +} + #[stable(feature = "rust1", since = "1.0.0")] impl Iterator for ReadDir { type Item = io::Result; @@ -2853,7 +2865,9 @@ impl DirEntry { /// /// On Windows this function is cheap to call (no extra system calls /// needed), but on Unix platforms this function is the equivalent of - /// calling `symlink_metadata` on the path. + /// calling `symlink_metadata` on the path. In that case, if the file has been renamed or + /// replaced since the `DirEntry` was created, this can fail or return the metadata of a + /// different file! /// /// # Examples /// @@ -2886,9 +2900,11 @@ impl DirEntry { /// /// # Platform-specific behavior /// - /// On Windows and most Unix platforms this function is free (no extra - /// system calls needed), but some Unix platforms may require the equivalent - /// call to `symlink_metadata` to learn about the target file type. + /// On Windows and most Unix platforms and file systems this function is free (no extra + /// system calls needed), but some Unix platforms or file systems may require the equivalent of + /// calling `symlink_metadata` to learn about the target file type. In that case, if the file + /// has been renamed or replaced since the `DirEntry` was created, this can fail or return the + /// file type of a different file! /// /// # Examples /// @@ -2970,6 +2986,9 @@ impl AsInner for DirEntry { /// On Windows, `DeleteFile` is used or `CreateFileW` and `SetInformationByHandle` for readonly files. /// Note that, this [may change in the future][changes]. /// +/// On Windows, symbolic links to directories cannot currently be removed with this function. +/// Use [`remove_dir`] for that purpose. +/// /// [changes]: io#platform-specific-behavior /// /// # Errors @@ -3038,7 +3057,7 @@ pub fn remove_file>(path: P) -> io::Result<()> { /// Ok(()) /// } /// ``` -#[doc(alias = "stat")] +#[doc(alias = "stat", alias = "GetFileAttributes", alias = "GetFileAttributesEx")] #[stable(feature = "rust1", since = "1.0.0")] #[cfg_attr(not(test), rustc_diagnostic_item = "fs_metadata")] pub fn metadata>(path: P) -> io::Result { @@ -3079,7 +3098,7 @@ pub fn metadata>(path: P) -> io::Result { /// Ok(()) /// } /// ``` -#[doc(alias = "lstat")] +#[doc(alias = "lstat", alias = "GetFileAttributes", alias = "GetFileAttributesEx")] #[stable(feature = "symlink_metadata", since = "1.1.0")] #[cfg_attr(not(test), rustc_diagnostic_item = "fs_symlink_metadata")] pub fn symlink_metadata>(path: P) -> io::Result { @@ -3135,6 +3154,76 @@ pub fn rename, Q: AsRef>(from: P, to: Q) -> io::Result<()> fs_imp::rename(from.as_ref(), to.as_ref()) } +/// Renames a file or directory to a new name, returning [`AlreadyExists`] if `to` already +/// exists. +/// +/// Unlike [`fs::rename`], the destination is never overwritten. The check happens as part of the +/// same operation, which avoids check-then-rename races. +/// +/// If the system crashes in the middle, it may be possible to end up with both copies of the file +/// existing, but it should not be possible to end up with zero copies, or with a half-written file. +/// +/// This will not work if the new name is on a different mount point. +/// +/// # Platform-specific behavior +/// +/// This function currently corresponds to: +/// +/// * [renameat2] with `RENAME_NOREPLACE` on Linux and Android +/// * `renamex_np` with `RENAME_EXCL` on Apple platforms +/// * [MoveFileExW] without the `MOVEFILE_REPLACE_EXISTING` flag, with a fallback to +/// `SetFileInformationByHandle`, on Windows +/// * `link` followed by `unlink` on other Unix platforms and on WASI +/// * Otherwise, this function returns [`Unsupported`] +/// +/// The `link` and `unlink` fallback is also used at runtime on Linux, Android and Apple platforms +/// when the kernel or the filesystem does not support the flag, so which of these applies can +/// differ between filesystems on the same machine. +/// +/// Note that, this [may change in the future][changes]. +/// +/// [changes]: io#platform-specific-behavior +/// [renameat2]: https://man7.org/linux/man-pages/man2/renameat2.2.html +/// [MoveFileExW]: https://learn.microsoft.com/en-us/windows/win32/api/winbase/nf-winbase-movefileexw +/// [`AlreadyExists`]: crate::io::ErrorKind::AlreadyExists +/// [`Unsupported`]: crate::io::ErrorKind::Unsupported +/// [`fs::rename`]: crate::fs::rename +/// +/// # Errors +/// +/// This function will return an error in the following situations, but is not +/// limited to just these cases: +/// +/// * `from` does not exist. +/// * The user lacks permissions to view contents. +/// * `from` and `to` are on separate filesystems. +/// * `to` already exists. +/// * The platform does not support this operation. +/// +/// # Examples +/// +/// ```no_run +/// #![feature(rename_noreplace)] +/// use std::fs; +/// use std::io::ErrorKind; +/// +/// fn main() -> std::io::Result<()> { +/// // Renames a.txt to b.txt, b.txt does not exist yet. +/// fs::rename_noreplace("a.txt", "b.txt")?; +/// +/// // b.txt exists now, so this fails instead of silently overwriting it. +/// let err = fs::rename_noreplace("c.txt", "b.txt").unwrap_err(); +/// assert_eq!(err.kind(), ErrorKind::AlreadyExists); +/// Ok(()) +/// } +/// ``` +#[doc(alias = "renameat2", alias = "renamex_np", alias = "MoveFileEx", alias = "RENAME_NOREPLACE")] +#[unstable(feature = "rename_noreplace", issue = "161427")] +#[cfg_attr(not(test), rustc_diagnostic_item = "fs_rename_noreplace")] +pub fn rename_noreplace, Q: AsRef>(from: P, to: Q) -> io::Result<()> { + fs_imp::rename_noreplace(from.as_ref(), to.as_ref()) +} + /// Copies the contents of one file to another. This function will also /// copy the permission bits of the original file to the destination file. /// @@ -3191,6 +3280,7 @@ pub fn rename, Q: AsRef>(from: P, to: Q) -> io::Result<()> /// } /// ``` #[doc(alias = "cp")] +#[doc(alias = "copy_file_range")] #[doc(alias = "CopyFile", alias = "CopyFileEx")] #[doc(alias = "fclonefileat", alias = "fcopyfile")] #[stable(feature = "rust1", since = "1.0.0")] @@ -3440,6 +3530,7 @@ pub fn create_dir>(path: P) -> io::Result<()> { /// Ok(()) /// } /// ``` +#[doc(alias = "mkdir", alias = "CreateDirectory")] #[stable(feature = "rust1", since = "1.0.0")] #[cfg_attr(not(test), rustc_diagnostic_item = "fs_create_dir_all")] pub fn create_dir_all>(path: P) -> io::Result<()> { @@ -3458,6 +3549,9 @@ pub fn create_dir_all>(path: P) -> io::Result<()> { /// and the `RemoveDirectory` function on Windows. /// Note that, this [may change in the future][changes]. /// +/// On Windows, this can also be used to remove symbolic links to directories. +/// On Unix, [`remove_file`] must be used for that purpose. +/// /// [changes]: io#platform-specific-behavior /// /// # Errors @@ -3496,7 +3590,9 @@ pub fn remove_dir>(path: P) -> io::Result<()> { /// carefully! /// /// This function does **not** follow symbolic links and it will simply remove the -/// symbolic link itself. +/// symbolic link itself. If `path` itself is a symbolic link to a directory, it will get +/// removed. If it is a symbolic link to a file, behavior is platform-specific: +/// currently, it will get removed on Unix-like systems but not on Windows. /// /// # Platform-specific behavior /// @@ -3552,6 +3648,7 @@ pub fn remove_dir>(path: P) -> io::Result<()> { /// Ok(()) /// } /// ``` +#[doc(alias = "rm")] #[stable(feature = "rust1", since = "1.0.0")] #[cfg_attr(not(test), rustc_diagnostic_item = "fs_remove_dir_all")] pub fn remove_dir_all>(path: P) -> io::Result<()> { diff --git a/library/std/src/fs/tests.rs b/library/std/src/fs/tests.rs index 54a6cf3904202..d611bac98f2e3 100644 --- a/library/std/src/fs/tests.rs +++ b/library/std/src/fs/tests.rs @@ -183,6 +183,7 @@ fn file_test_io_seek_and_write() { target_os = "freebsd", target_os = "fuchsia", target_os = "hurd", + target_os = "haiku", target_os = "illumos", target_os = "linux", target_os = "netbsd", @@ -217,6 +218,7 @@ fn file_lock_multiple_shared() { target_os = "freebsd", target_os = "fuchsia", target_os = "hurd", + target_os = "haiku", target_os = "illumos", target_os = "linux", target_os = "netbsd", @@ -252,6 +254,7 @@ fn file_lock_blocking() { target_os = "freebsd", target_os = "fuchsia", target_os = "hurd", + target_os = "haiku", target_os = "illumos", target_os = "linux", target_os = "netbsd", @@ -284,6 +287,7 @@ fn file_lock_drop() { target_os = "freebsd", target_os = "fuchsia", target_os = "hurd", + target_os = "haiku", target_os = "illumos", target_os = "linux", target_os = "netbsd", @@ -318,6 +322,7 @@ fn file_lock_dup() { target_os = "freebsd", target_os = "fuchsia", target_os = "hurd", + target_os = "haiku", target_os = "illumos", target_os = "linux", target_os = "netbsd", @@ -340,7 +345,7 @@ fn file_lock_double() { // The behavior here differs between Windows and Unix: on Windows, f1 holds both locks; // on Unix, the lock got downgraded so f1 only holds the shared lock. assert_matches!(f2.try_lock(), Err(TryLockError::WouldBlock)); - if cfg!(windows) { + if cfg!(any(windows, target_os = "haiku")) { assert_matches!(f2.try_lock_shared(), Err(TryLockError::WouldBlock)); } else { check!(f2.try_lock_shared()); @@ -847,102 +852,175 @@ fn file_test_windows_fileext_trait_case_1() { #[test] #[cfg(windows)] fn file_test_windows_fileext_trait_case_2() { + use crate::cell::RefCell; use crate::os::windows::fs::FileExt; // Test when seek_read(), seek_write() return Ok(0) struct MockFile { expected_offset: u64, + read_calls: RefCell, + write_calls: RefCell, + } + + impl MockFile { + fn new(expected_offset: u64) -> Self { + Self { expected_offset, read_calls: RefCell::new(0), write_calls: RefCell::new(0) } + } } impl FileExt for MockFile { fn seek_read(&self, _buf: &mut [u8], offset: u64) -> io::Result { + *self.read_calls.borrow_mut() += 1; assert_eq!(offset, self.expected_offset); Ok(0) } fn seek_write(&self, _buf: &[u8], offset: u64) -> io::Result { + *self.write_calls.borrow_mut() += 1; assert_eq!(offset, self.expected_offset); Ok(0) } } - { - let mock_file = MockFile { expected_offset: 0 }; - let mut buf = [0; 256]; - assert_eq!( - mock_file.seek_read_exact(&mut buf, 0).unwrap_err().kind(), - io::ErrorKind::UnexpectedEof - ); - assert_eq!(mock_file.seek_write_all(&buf, 0).unwrap_err().kind(), io::ErrorKind::WriteZero); - } - - { - let mock_file = MockFile { expected_offset: 420 }; - let mut buf = [0; 256]; - assert_eq!( - mock_file.seek_read_exact(&mut buf, 420).unwrap_err().kind(), - io::ErrorKind::UnexpectedEof - ); - assert_eq!( - mock_file.seek_write_all(&buf, 420).unwrap_err().kind(), - io::ErrorKind::WriteZero - ); + let mut buf = [0; 256]; + for offset in [0, 1, 420] { + { + let mock_file = MockFile::new(offset); + assert_eq!( + mock_file.seek_read_exact(&mut buf, offset).unwrap_err().kind(), + ErrorKind::UnexpectedEof + ); + assert_eq!(*mock_file.read_calls.borrow(), 1); + assert_eq!(*mock_file.write_calls.borrow(), 0); + } + { + let mock_file = MockFile::new(offset); + assert_eq!( + mock_file.seek_write_all(&buf, offset).unwrap_err().kind(), + ErrorKind::WriteZero + ); + assert_eq!(*mock_file.read_calls.borrow(), 0); + assert_eq!(*mock_file.write_calls.borrow(), 1); + } } } #[test] #[cfg(windows)] fn file_test_windows_fileext_trait_case_3() { + use crate::cell::RefCell; use crate::os::windows::fs::FileExt; - // Test that Err other than io::ErrorKind::Interrupted are propagated up. + // Test that Err are propagated up. struct MockFile { expected_offset: u64, + read_calls: RefCell, + write_calls: RefCell, + } + + impl MockFile { + fn new(expected_offset: u64) -> Self { + Self { expected_offset, read_calls: RefCell::new(0), write_calls: RefCell::new(0) } + } } impl FileExt for MockFile { fn seek_read(&self, _buf: &mut [u8], offset: u64) -> io::Result { + *self.read_calls.borrow_mut() += 1; assert_eq!(offset, self.expected_offset); - Err(io::Error::new(io::ErrorKind::PermissionDenied, "seek_read")) + Err(io::Error::new(ErrorKind::PermissionDenied, "seek_read")) } fn seek_write(&self, _buf: &[u8], offset: u64) -> io::Result { + *self.write_calls.borrow_mut() += 1; assert_eq!(offset, self.expected_offset); - Err(io::Error::new(io::ErrorKind::ConnectionRefused, "seek_write")) + Err(io::Error::new(ErrorKind::ConnectionRefused, "seek_write")) } } - { - let mock_file = MockFile { expected_offset: 0 }; - let mut buf = [0; 256]; - assert_eq!( - mock_file.seek_read_exact(&mut buf, 0).unwrap_err().kind(), - io::ErrorKind::PermissionDenied - ); - assert_eq!( - mock_file.seek_write_all(&buf, 0).unwrap_err().kind(), - io::ErrorKind::ConnectionRefused - ); + let mut buf = [0; 256]; + for offset in [0, 1, 420] { + { + let mock_file = MockFile::new(offset); + assert_eq!( + mock_file.seek_read_exact(&mut buf, offset).unwrap_err().kind(), + ErrorKind::PermissionDenied + ); + assert_eq!(*mock_file.read_calls.borrow(), 1); + assert_eq!(*mock_file.write_calls.borrow(), 0); + } + { + let mock_file = MockFile::new(offset); + assert_eq!( + mock_file.seek_write_all(&buf, offset).unwrap_err().kind(), + ErrorKind::ConnectionRefused + ); + assert_eq!(*mock_file.read_calls.borrow(), 0); + assert_eq!(*mock_file.write_calls.borrow(), 1); + } } +} - { - let mock_file = MockFile { expected_offset: 420 }; - let mut buf = [0; 256]; - assert_eq!( - mock_file.seek_read_exact(&mut buf, 420).unwrap_err().kind(), - io::ErrorKind::PermissionDenied - ); - assert_eq!( - mock_file.seek_write_all(&buf, 420).unwrap_err().kind(), - io::ErrorKind::ConnectionRefused - ); +#[test] +#[cfg(windows)] +fn file_test_windows_fileext_trait_case_4() { + use crate::cell::RefCell; + use crate::os::windows::fs::FileExt; + + // Test that Err ErrorKind::Interrupted are propagated up (unlike Unix versions) + struct MockFile { + expected_offset: u64, + read_calls: RefCell, + write_calls: RefCell, + } + + impl MockFile { + fn new(expected_offset: u64) -> Self { + Self { expected_offset, read_calls: RefCell::new(0), write_calls: RefCell::new(0) } + } + } + + impl FileExt for MockFile { + fn seek_read(&self, _buf: &mut [u8], offset: u64) -> io::Result { + *self.read_calls.borrow_mut() += 1; + assert_eq!(offset, self.expected_offset); + Err(io::Error::new(ErrorKind::Interrupted, "seek_read")) + } + + fn seek_write(&self, _buf: &[u8], offset: u64) -> io::Result { + *self.write_calls.borrow_mut() += 1; + assert_eq!(offset, self.expected_offset); + Err(io::Error::new(ErrorKind::Interrupted, "seek_write")) + } + } + + let mut buf = [0; 256]; + for offset in [0, 1, 420] { + { + let mock_file = MockFile::new(offset); + assert_eq!( + mock_file.seek_read_exact(&mut buf, offset).unwrap_err().kind(), + ErrorKind::Interrupted + ); + assert_eq!(*mock_file.read_calls.borrow(), 1); + assert_eq!(*mock_file.write_calls.borrow(), 0); + } + { + let mock_file = MockFile::new(offset); + assert_eq!( + mock_file.seek_write_all(&buf, offset).unwrap_err().kind(), + ErrorKind::Interrupted + ); + assert_eq!(*mock_file.read_calls.borrow(), 0); + assert_eq!(*mock_file.write_calls.borrow(), 1); + } } - // FIXME: Cover io::ErrorKind::Interrupted, but don't infinite loop ;) } #[test] #[cfg(windows)] -fn file_test_windows_fileext_trait_case_4() { +fn file_test_windows_fileext_trait_case_5() { + use crate::cell::RefCell; use crate::os::windows::fs::FileExt; const MSG: &[u8] = @@ -952,10 +1030,19 @@ fn file_test_windows_fileext_trait_case_4() { // seek_write(), respectively. struct MockFile { expected_offset: u64, + read_calls: RefCell, + write_calls: RefCell, + } + + impl MockFile { + fn new(expected_offset: u64) -> Self { + Self { expected_offset, read_calls: RefCell::new(0), write_calls: RefCell::new(0) } + } } impl FileExt for MockFile { fn seek_read(&self, buf: &mut [u8], offset: u64) -> io::Result { + *self.read_calls.borrow_mut() += 1; assert_eq!(offset, self.expected_offset); assert_eq!(buf.len(), MSG.len()); assert_eq!(buf, &[0; MSG.len()]); @@ -964,6 +1051,7 @@ fn file_test_windows_fileext_trait_case_4() { } fn seek_write(&self, buf: &[u8], offset: u64) -> io::Result { + *self.write_calls.borrow_mut() += 1; assert_eq!(offset, self.expected_offset); assert_eq!(buf.len(), MSG.len()); assert_eq!(buf, MSG); @@ -971,26 +1059,28 @@ fn file_test_windows_fileext_trait_case_4() { } } - { - let mock_file = MockFile { expected_offset: 0 }; - let mut buf = [0; MSG.len()]; - check!(mock_file.seek_read_exact(&mut buf, 0)); - assert_eq!(&buf, MSG); - check!(mock_file.seek_write_all(&buf, 0)); - } - - { - let mock_file = MockFile { expected_offset: 420 }; - let mut buf = [0; MSG.len()]; - check!(mock_file.seek_read_exact(&mut buf, 420)); - assert_eq!(&buf, MSG); - check!(mock_file.seek_write_all(&buf, 420)); + for offset in [0, 1, 420] { + { + let mock_file = MockFile::new(offset); + let mut buf = [0; MSG.len()]; + check!(mock_file.seek_read_exact(&mut buf, offset)); + assert_eq!(&buf, MSG); + assert_eq!(*mock_file.read_calls.borrow(), 1); + assert_eq!(*mock_file.write_calls.borrow(), 0); + } + { + let mock_file = MockFile::new(offset); + check!(mock_file.seek_write_all(MSG, offset)); + assert_eq!(*mock_file.read_calls.borrow(), 0); + assert_eq!(*mock_file.write_calls.borrow(), 1); + } } } #[test] #[cfg(windows)] -fn file_test_windows_fileext_trait_case_5() { +fn file_test_windows_fileext_trait_case_6() { + use crate::cell::RefCell; use crate::os::windows::fs::FileExt; const MSG: &[u8] = @@ -999,36 +1089,47 @@ fn file_test_windows_fileext_trait_case_5() { // Test pathological case where seek_read(), seek_write() only do 1 byte per call, return Ok(1) struct MockFile { base_offset: u64, + read_calls: RefCell, + write_calls: RefCell, + } + + impl MockFile { + fn new(base_offset: u64) -> Self { + Self { base_offset, read_calls: RefCell::new(0), write_calls: RefCell::new(0) } + } } impl FileExt for MockFile { fn seek_read(&self, buf: &mut [u8], offset: u64) -> io::Result { + *self.read_calls.borrow_mut() += 1; let offset = (offset - self.base_offset) as usize; buf[0..1].copy_from_slice(&MSG[offset..offset + 1]); Ok(1) } fn seek_write(&self, buf: &[u8], offset: u64) -> io::Result { + *self.write_calls.borrow_mut() += 1; let offset = (offset - self.base_offset) as usize; assert_eq!(buf[0..1], MSG[offset..offset + 1]); Ok(1) } } - { - let mock_file = MockFile { base_offset: 0 }; - let mut buf = [0; MSG.len()]; - check!(mock_file.seek_read_exact(&mut buf, 0)); - assert_eq!(&buf, MSG); - check!(mock_file.seek_write_all(&buf, 0)); - } - - { - let mock_file = MockFile { base_offset: 420 }; - let mut buf = [0; MSG.len()]; - check!(mock_file.seek_read_exact(&mut buf, 420)); - assert_eq!(&buf, MSG); - check!(mock_file.seek_write_all(&buf, 420)); + for offset in [0, 1, 420] { + { + let mock_file = MockFile::new(offset); + let mut buf = [0; MSG.len()]; + check!(mock_file.seek_read_exact(&mut buf, offset)); + assert_eq!(&buf, MSG); + assert_eq!(*mock_file.read_calls.borrow(), MSG.len()); + assert_eq!(*mock_file.write_calls.borrow(), 0); + } + { + let mock_file = MockFile::new(offset); + check!(mock_file.seek_write_all(MSG, offset)); + assert_eq!(*mock_file.read_calls.borrow(), 0); + assert_eq!(*mock_file.write_calls.borrow(), MSG.len()); + } } } @@ -1315,7 +1416,7 @@ fn recursive_rmdir() { } #[test] -fn recursive_rmdir_of_symlink() { +fn recursive_rmdir_of_dir_symlink() { // test we do not recursively delete a symlink but only dirs. let tmpdir = tmpdir(); let link = tmpdir.join("d1"); @@ -1342,8 +1443,6 @@ fn recursive_rmdir_of_file_fails() { } #[test] -// only Windows makes a distinction between file and directory symlinks. -#[cfg(windows)] fn recursive_rmdir_of_file_symlink() { let tmpdir = tmpdir(); if !got_symlink_permission(&tmpdir) { @@ -1354,9 +1453,12 @@ fn recursive_rmdir_of_file_symlink() { let f2 = tmpdir.join("f2"); check!(check!(File::create(&f1)).write(b"foo")); check!(symlink_file(&f1, &f2)); - match fs::remove_dir_all(&f2) { - Ok(..) => panic!("wanted a failure"), - Err(..) => {} + let res = fs::remove_dir_all(&f2); + // Windows and Unix behavior differs here. + if cfg!(windows) { + let _err = res.unwrap_err(); + } else { + let _ok = res.unwrap(); } } @@ -1730,7 +1832,6 @@ fn fchmod_works() { check!(file.set_permissions(p)); } -#[cfg(not(target_os = "android"))] #[test] fn fchmodat_works() { let tmpdir = tmpdir(); @@ -2664,6 +2765,98 @@ fn test_rename_symlink() { assert_eq!(fs::read_link(&dest).unwrap().as_path(), not_exist); } +#[test] +fn test_rename_noreplace() { + let tmpdir = tmpdir(); + let source_path = tmpdir.join("source_file.txt"); + let target_path = tmpdir.join("target_file.txt"); + + fs::write(&source_path, b"source hello world").unwrap(); + + fs::rename_noreplace(&source_path, &target_path).unwrap(); + assert!(!source_path.exists()); + assert_eq!(fs::read(&target_path).unwrap(), b"source hello world"); +} + +#[test] +fn test_rename_noreplace_existing_file() { + let tmpdir = tmpdir(); + let source_path = tmpdir.join("source_file.txt"); + let target_path = tmpdir.join("target_file.txt"); + + fs::write(&source_path, b"source hello world").unwrap(); + fs::write(&target_path, b"target hello world").unwrap(); + + let err = fs::rename_noreplace(&source_path, &target_path).unwrap_err(); + assert_eq!(err.kind(), ErrorKind::AlreadyExists); + // Make sure the failed rename left both files untouched. + assert_eq!(fs::read(&source_path).unwrap(), b"source hello world"); + assert_eq!(fs::read(&target_path).unwrap(), b"target hello world"); +} + +#[test] +fn test_rename_noreplace_directory_to_empty_directory() { + // Renaming a directory over an existing empty directory should fail; + // plain `rename` would succeed here. + let tmpdir = tmpdir(); + let source_path = tmpdir.join("source_directory"); + let target_path = tmpdir.join("target_directory"); + + fs::create_dir(&source_path).unwrap(); + fs::create_dir(&target_path).unwrap(); + + let err = fs::rename_noreplace(&source_path, &target_path).unwrap_err(); + assert_matches!( + err.kind(), + // A native no-replace rename returns `AlreadyExists`. The `link`/`unlink` + // fallback cannot rename directories and returns `PermissionDenied`. + ErrorKind::AlreadyExists | ErrorKind::PermissionDenied, + "Expected AlreadyExists or PermissionDenied error, got {err}" + ); +} + +#[test] +#[cfg(any(target_os = "linux", target_os = "android", target_vendor = "apple", windows))] +fn test_rename_noreplace_directory() { + let tmpdir = tmpdir(); + let source_path = tmpdir.join("source_directory"); + let target_path = tmpdir.join("target_directory"); + + fs::create_dir(&source_path).unwrap(); + fs::write(source_path.join("file.txt"), b"hello world").unwrap(); + + match fs::rename_noreplace(&source_path, &target_path) { + Ok(()) => { + assert!(!source_path.exists()); + assert_eq!(fs::read(target_path.join("file.txt")).unwrap(), b"hello world"); + } + // Without a native no-replace rename (Linux before 3.15), Unix + // platforms fall back to `link`/`unlink`, and it rejects directories. + Err(err) if cfg!(unix) && err.kind() == ErrorKind::PermissionDenied => { + assert!(source_path.exists()); + assert!(!target_path.exists()); + } + Err(err) => panic!("unexpected error: {err:?}"), + } +} + +#[test] +fn test_rename_noreplace_symlink() { + let tmpdir = tmpdir(); + if !got_symlink_permission(&tmpdir) { + return; + }; + + let original = tmpdir.join("original"); + let dest = tmpdir.join("dest"); + let not_exist = Path::new("does not exist"); + + symlink_file(not_exist, &original).unwrap(); + fs::rename_noreplace(&original, &dest).unwrap(); + // Make sure that renaming `original` to `dest` preserves the symlink. + assert_eq!(fs::read_link(&dest).unwrap().as_path(), not_exist); +} + #[test] #[cfg(windows)] fn test_rename_junction() { @@ -3075,10 +3268,6 @@ fn test_dir_rename_file() { assert_eq!(b"bar", &buf); } -// FIXME: re-enable once QNX fixes TOCTOU bug for fs::remove_dir -// Note that it may get fixed in QNX 8 in a future libc release -// ... https://github.com/rust-lang/rust/issues/153781 -#[cfg_attr(any(target_os = "nto", target_os = "qnx"), ignore)] #[test] fn test_dir_remove_dir() { let tmpdir = tmpdir(); @@ -3148,6 +3337,19 @@ fn test_dir_metadata() { assert!(metadata.is_symlink()); } +#[test] +fn test_read_dir_dir() { + let tmpdir = tmpdir(); + check!(fs::write(tmpdir.path().join("file.txt"), b"hello")); + + let read_dir = check!(fs::read_dir(tmpdir.path())); + let dir = check!(read_dir.dir()); + let mut f = check!(dir.open_file("file.txt")); + let mut data = vec![]; + check!(f.read_to_end(&mut data)); + assert_eq!(data, b"hello"); +} + fn root_test_dir(what: &str) -> PathBuf { crate::env::current_dir().unwrap().ancestors().last().unwrap().join(what) } diff --git a/library/std/src/io/stdio.rs b/library/std/src/io/stdio.rs index 0848f97e69d32..f72a55366bfdd 100644 --- a/library/std/src/io/stdio.rs +++ b/library/std/src/io/stdio.rs @@ -95,27 +95,6 @@ const fn stderr_raw() -> StderrRaw { StderrRaw(stdio::Stderr::new()) } -#[cfg(windows)] -impl StdoutRaw { - /// Starts a new lock session: stream state cached per lock session (the - /// handle and its console mode) is re-queried at the next write, so that - /// changing the process stdio handles (e.g. with `SetStdHandle`) between - /// lock sessions keeps working. - #[inline] - fn refresh(&mut self) { - self.0.refresh(); - } -} - -#[cfg(windows)] -impl StderrRaw { - /// See `StdoutRaw::refresh`. - #[inline] - fn refresh(&mut self) { - self.0.refresh(); - } -} - impl Read for StdinRaw { fn read(&mut self, buf: &mut [u8]) -> io::Result { handle_ebadf(self.0.read(buf), || Ok(0)) @@ -615,9 +594,8 @@ impl fmt::Debug for StdinLock<'_> { /// /// ### Note: Windows Portability Considerations /// -/// When operating in a console, the Windows implementation of this stream does not support -/// non-UTF-8 byte sequences. Attempting to write bytes that are not valid UTF-8 will return -/// an error. +/// When operating in a console, the Windows implementation of this stream decodes bytes +/// as UTF-8, replacing malformed sequences with the Unicode replacement character (U+FFFD). /// /// In a process with a detached console, such as one using /// `#![windows_subsystem = "windows"]`, or in a child process spawned from such a process, @@ -648,9 +626,8 @@ pub struct Stdout { /// /// ### Note: Windows Portability Considerations /// -/// When operating in a console, the Windows implementation of this stream does not support -/// non-UTF-8 byte sequences. Attempting to write bytes that are not valid UTF-8 will return -/// an error. +/// When operating in a console, the Windows implementation of this stream decodes bytes +/// as UTF-8, replacing malformed sequences with the Unicode replacement character (U+FFFD). /// /// In a process with a detached console, such as one using /// `#![windows_subsystem = "windows"]`, or in a child process spawned from such a process, @@ -680,9 +657,8 @@ static STDOUT: OnceLock>>> = OnceLoc /// /// ### Note: Windows Portability Considerations /// -/// When operating in a console, the Windows implementation of this stream does not support -/// non-UTF-8 byte sequences. Attempting to write bytes that are not valid UTF-8 will return -/// an error. +/// When operating in a console, the Windows implementation of this stream decodes bytes +/// as UTF-8, replacing malformed sequences with the Unicode replacement character (U+FFFD). /// /// In a process with a detached console, such as one using /// `#![windows_subsystem = "windows"]`, or in a child process spawned from such a process, @@ -790,19 +766,7 @@ impl Stdout { // Locks this handle with 'static lifetime. This depends on the // implementation detail that the underlying `ReentrantMutex` is // static. - let lock = StdoutLock { inner: self.inner.lock() }; - // A new lock session begins, so let Windows re-query cached stream - // state at the next write; other platforms cache nothing, and skip - // this entirely so their `lock()` is unchanged. Skipped also if the - // cell is already borrowed (a nested lock during an ongoing write); - // such a lock is part of the outer session anyway. - #[cfg(windows)] - { - if let Ok(mut w) = lock.inner.try_borrow_mut() { - w.get_mut().refresh(); - } - } - lock + StdoutLock { inner: self.inner.lock() } } } @@ -915,9 +879,8 @@ impl fmt::Debug for StdoutLock<'_> { /// /// ### Note: Windows Portability Considerations /// -/// When operating in a console, the Windows implementation of this stream does not support -/// non-UTF-8 byte sequences. Attempting to write bytes that are not valid UTF-8 will return -/// an error. +/// When operating in a console, the Windows implementation of this stream decodes bytes +/// as UTF-8, replacing malformed sequences with the Unicode replacement character (U+FFFD). /// /// In a process with a detached console, such as one using /// `#![windows_subsystem = "windows"]`, or in a child process spawned from such a process, @@ -936,9 +899,8 @@ pub struct Stderr { /// /// ### Note: Windows Portability Considerations /// -/// When operating in a console, the Windows implementation of this stream does not support -/// non-UTF-8 byte sequences. Attempting to write bytes that are not valid UTF-8 will return -/// an error. +/// When operating in a console, the Windows implementation of this stream decodes bytes +/// as UTF-8, replacing malformed sequences with the Unicode replacement character (U+FFFD). /// /// In a process with a detached console, such as one using /// `#![windows_subsystem = "windows"]`, or in a child process spawned from such a process, @@ -957,9 +919,8 @@ pub struct StderrLock<'a> { /// /// ### Note: Windows Portability Considerations /// -/// When operating in a console, the Windows implementation of this stream does not support -/// non-UTF-8 byte sequences. Attempting to write bytes that are not valid UTF-8 will return -/// an error. +/// When operating in a console, the Windows implementation of this stream decodes bytes +/// as UTF-8, replacing malformed sequences with the Unicode replacement character (U+FFFD). /// /// In a process with a detached console, such as one using /// `#![windows_subsystem = "windows"]`, or in a child process spawned from such a process, @@ -1034,15 +995,7 @@ impl Stderr { // Locks this handle with 'static lifetime. This depends on the // implementation detail that the underlying `ReentrantMutex` is // static. - let lock = StderrLock { inner: self.inner.lock() }; - // See `Stdout::lock`: a new lock session begins (Windows only). - #[cfg(windows)] - { - if let Ok(mut w) = lock.inner.try_borrow_mut() { - w.refresh(); - } - } - lock + StderrLock { inner: self.inner.lock() } } } diff --git a/library/std/src/lib.rs b/library/std/src/lib.rs index b1e9f3880cc81..8253c161e6274 100644 --- a/library/std/src/lib.rs +++ b/library/std/src/lib.rs @@ -318,8 +318,8 @@ // Library features (core): // tidy-alphabetical-start #![feature(borrowed_buf_init)] -#![feature(bstr)] -#![feature(bstr_internals)] +#![feature(byte_str)] +#![feature(byte_str_internals)] #![feature(c_size_t)] #![feature(can_vector)] #![feature(cast_maybe_uninit)] @@ -359,6 +359,7 @@ #![feature(io_error_uncategorized)] #![feature(io_slice_as_bytes)] #![feature(ip)] +#![feature(ipv6_hop_limit)] #![feature(iter_advance_by)] #![feature(iter_next_chunk)] #![feature(maybe_dangling)] @@ -381,7 +382,6 @@ #![feature(slice_internals)] #![feature(slice_ptr_get)] #![feature(slice_range)] -#![feature(slice_split_once)] #![feature(std_internals)] #![feature(str_internals)] #![feature(sync_unsafe_cell)] @@ -625,8 +625,8 @@ pub mod f64; pub mod thread; pub mod ascii; pub mod backtrace; -#[unstable(feature = "bstr", issue = "134915")] -pub mod bstr; +#[unstable(feature = "byte_str", issue = "134915")] +pub mod byte_str; pub mod collections; pub mod env; pub mod error; @@ -711,6 +711,8 @@ pub mod arch { pub use std_detect::is_arm_feature_detected; #[stable(feature = "stdarch_loongarch_feature", since = "1.89.0")] pub use std_detect::is_loongarch_feature_detected; + #[unstable(feature = "stdarch_powerpc_feature_detection", issue = "111191")] + pub use std_detect::is_powerpc_feature_detected; #[stable(feature = "riscv_ratified", since = "1.78.0")] pub use std_detect::is_riscv_feature_detected; #[stable(feature = "stdarch_s390x_feature_detection", since = "1.93.0")] @@ -719,8 +721,6 @@ pub mod arch { pub use std_detect::is_x86_feature_detected; #[unstable(feature = "stdarch_mips_feature_detection", issue = "111188")] pub use std_detect::{is_mips_feature_detected, is_mips64_feature_detected}; - #[unstable(feature = "stdarch_powerpc_feature_detection", issue = "111191")] - pub use std_detect::{is_powerpc_feature_detected, is_powerpc64_feature_detected}; } // This was stabilized in the crate root so we have to keep it there. diff --git a/library/std/src/net/tcp.rs b/library/std/src/net/tcp.rs index e5c2522a837fb..7f3b00a315862 100644 --- a/library/std/src/net/tcp.rs +++ b/library/std/src/net/tcp.rs @@ -605,6 +605,46 @@ impl TcpStream { self.0.ttl() } + /// Sets the value for the `IPV6_UNICAST_HOPS` option on this socket. + /// + /// This value sets the unicast hop limit field that is used in every packet + /// sent from this socket. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(ipv6_hop_limit)] + /// use std::net::TcpStream; + /// + /// let stream = TcpStream::connect("[::1]:12345") + /// .expect("Couldn't connect to the server..."); + /// stream.set_hop_limit_v6(88).expect("set_hop_limit_v6 call failed"); + /// ``` + #[unstable(feature = "ipv6_hop_limit", issue = "139166")] + pub fn set_hop_limit_v6(&self, limit: u8) -> io::Result<()> { + self.0.set_hop_limit_v6(limit) + } + + /// Gets the value of the `IPV6_UNICAST_HOPS` option on this socket. + /// + /// For more information about this option, see [`TcpStream::set_hop_limit_v6`]. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(ipv6_hop_limit)] + /// use std::net::TcpStream; + /// + /// let stream = TcpStream::connect("[::1]:12345") + /// .expect("Couldn't connect to the server..."); + /// stream.set_hop_limit_v6(88).expect("set_hop_limit_v6 call failed"); + /// assert_eq!(stream.hop_limit_v6().unwrap(), 88); + /// ``` + #[unstable(feature = "ipv6_hop_limit", issue = "139166")] + pub fn hop_limit_v6(&self) -> io::Result { + self.0.hop_limit_v6() + } + /// Gets the value of the `SO_ERROR` option on this socket. /// /// This will retrieve the stored error in the underlying socket, clearing @@ -1032,6 +1072,44 @@ impl TcpListener { self.0.ttl() } + /// Sets the value for the `IPV6_UNICAST_HOPS` option on this socket. + /// + /// This value sets the unicast hop limit field that is used in every packet + /// sent from this socket. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(ipv6_hop_limit)] + /// use std::net::TcpListener; + /// + /// let listener = TcpListener::bind("[::1]:12345").unwrap(); + /// listener.set_hop_limit_v6(88).expect("set_hop_limit_v6 call failed"); + /// ``` + #[unstable(feature = "ipv6_hop_limit", issue = "139166")] + pub fn set_hop_limit_v6(&self, limit: u8) -> io::Result<()> { + self.0.set_hop_limit_v6(limit) + } + + /// Gets the value of the `IPV6_UNICAST_HOPS` option on this socket. + /// + /// For more information about this option, see [`TcpListener::set_hop_limit_v6`]. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(ipv6_hop_limit)] + /// use std::net::TcpListener; + /// + /// let listener = TcpListener::bind("[::1]:12345").unwrap(); + /// listener.set_hop_limit_v6(88).expect("set_hop_limit_v6 call failed"); + /// assert_eq!(listener.hop_limit_v6().unwrap(), 88); + /// ``` + #[unstable(feature = "ipv6_hop_limit", issue = "139166")] + pub fn hop_limit_v6(&self) -> io::Result { + self.0.hop_limit_v6() + } + #[stable(feature = "net2_mutators", since = "1.9.0")] #[deprecated(since = "1.16.0", note = "this option can only be set before the socket is bound")] #[allow(missing_docs)] diff --git a/library/std/src/net/tcp/tests.rs b/library/std/src/net/tcp/tests.rs index 45a512cb9b4c7..58de3d478caa0 100644 --- a/library/std/src/net/tcp/tests.rs +++ b/library/std/src/net/tcp/tests.rs @@ -894,6 +894,23 @@ fn ttl() { assert_eq!(ttl, t!(stream.ttl())); } +#[test] +#[cfg_attr(target_env = "sgx", ignore)] +#[cfg(not(all(target_os = "wasi", any(target_env = "p2", target_env = "p3"))))] +fn hop_limit() { + let hlim = 100; + + let listener = t!(TcpListener::bind(LOCALHOST_IP6)); + + t!(listener.set_hop_limit_v6(hlim)); + assert_eq!(hlim, t!(listener.hop_limit_v6())); + + let stream = t!(TcpStream::connect(t!(listener.local_addr()))); + + t!(stream.set_hop_limit_v6(hlim)); + assert_eq!(hlim, t!(stream.hop_limit_v6())); +} + #[test] #[cfg_attr(target_env = "sgx", ignore)] fn set_nonblocking() { diff --git a/library/std/src/net/udp.rs b/library/std/src/net/udp.rs index 6448c9f2e0091..914c400fb05f1 100644 --- a/library/std/src/net/udp.rs +++ b/library/std/src/net/udp.rs @@ -589,6 +589,82 @@ impl UdpSocket { self.0.ttl() } + /// Sets the value for the `IPV6_UNICAST_HOPS` option on this socket. + /// + /// This value sets the unicast hop limit field that is used in every packet + /// sent from this socket. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(ipv6_hop_limit)] + /// use std::net::UdpSocket; + /// + /// let socket = UdpSocket::bind("[::1]:12345").expect("couldn't bind to address"); + /// socket.set_hop_limit_v6(88).expect("set_hop_limit_v6 call failed"); + /// ``` + #[unstable(feature = "ipv6_hop_limit", issue = "139166")] + pub fn set_hop_limit_v6(&self, limit: u8) -> io::Result<()> { + self.0.set_hop_limit_v6(limit) + } + + /// Gets the value of the `IPV6_UNICAST_HOPS` option on this socket. + /// + /// For more information about this option, see [`UdpSocket::set_hop_limit_v6`]. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(ipv6_hop_limit)] + /// use std::net::UdpSocket; + /// + /// let socket = UdpSocket::bind("[::1]:12345").expect("couldn't bind to address"); + /// socket.set_hop_limit_v6(88).expect("set_hop_limit_v6 call failed"); + /// assert_eq!(socket.hop_limit_v6().unwrap(), 88); + /// ``` + #[unstable(feature = "ipv6_hop_limit", issue = "139166")] + pub fn hop_limit_v6(&self) -> io::Result { + self.0.hop_limit_v6() + } + + /// Sets the value for the `IPV6_MULTICAST_HOPS` option on this socket. + /// + /// This value sets the hop limit field for outgoing multicast packets + /// sent from this socket. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(ipv6_hop_limit)] + /// use std::net::UdpSocket; + /// + /// let socket = UdpSocket::bind("[::1]:12345").expect("couldn't bind to address"); + /// socket.set_multicast_hop_limit_v6(88).expect("set_multicast_hop_limit_v6 call failed"); + /// ``` + #[unstable(feature = "ipv6_hop_limit", issue = "139166")] + pub fn set_multicast_hop_limit_v6(&self, limit: u8) -> io::Result<()> { + self.0.set_multicast_hop_limit_v6(limit) + } + + /// Gets the value of the `IPV6_MULTICAST_HOPS` option on this socket. + /// + /// For more information about this option, see [`UdpSocket::set_multicast_hop_limit_v6`]. + /// + /// # Examples + /// + /// ```no_run + /// #![feature(ipv6_hop_limit)] + /// use std::net::UdpSocket; + /// + /// let socket = UdpSocket::bind("[::1]:12345").expect("couldn't bind to address"); + /// socket.set_multicast_hop_limit_v6(88).expect("set_multicast_hop_limit_v6 call failed"); + /// assert_eq!(socket.multicast_hop_limit_v6().unwrap(), 88); + /// ``` + #[unstable(feature = "ipv6_hop_limit", issue = "139166")] + pub fn multicast_hop_limit_v6(&self) -> io::Result { + self.0.multicast_hop_limit_v6() + } + /// Executes an operation of the `IP_ADD_MEMBERSHIP` type. /// /// This function specifies a new multicast group for this socket to join. diff --git a/library/std/src/net/udp/tests.rs b/library/std/src/net/udp/tests.rs index 38d6dab80f64e..89ec91122c5fb 100644 --- a/library/std/src/net/udp/tests.rs +++ b/library/std/src/net/udp/tests.rs @@ -353,6 +353,17 @@ fn ttl() { assert_eq!(ttl, t!(stream.ttl())); } +#[test] +#[cfg(not(all(target_os = "wasi", any(target_env = "p2", target_env = "p3"))))] +fn hop_limit() { + let hlim = 100; + + let stream = t!(UdpSocket::bind(LOCALHOST_IP6)); + + t!(stream.set_hop_limit_v6(hlim)); + assert_eq!(hlim, t!(stream.hop_limit_v6())); +} + #[test] fn set_nonblocking() { each_ip(&mut |socket, _| { diff --git a/library/std/src/os/unix/net/addr.rs b/library/std/src/os/unix/net/addr.rs index 0520da62ffae9..b03b359cdea91 100644 --- a/library/std/src/os/unix/net/addr.rs +++ b/library/std/src/os/unix/net/addr.rs @@ -1,4 +1,4 @@ -use crate::bstr::ByteStr; +use crate::byte_str::ByteStr; use crate::ffi::OsStr; #[cfg(any(doc, target_os = "android", target_os = "linux", target_os = "cygwin"))] use crate::os::net::linux_ext; diff --git a/library/std/src/os/unix/process.rs b/library/std/src/os/unix/process.rs index 9fa731de82691..edd7d8dfb092d 100644 --- a/library/std/src/os/unix/process.rs +++ b/library/std/src/os/unix/process.rs @@ -213,6 +213,36 @@ pub impl(self) trait CommandExt { /// If no directory has been specified with [`process::Command::current_dir`], this will set the /// directory to `/`, to avoid leaving the current directory outside the chroot. (This is an /// intentional difference from the underlying `chroot` system call.) + /// + /// This typically requires privileges, such as root or a specific capability. + /// + /// Note that `chroot` is not guaranteed to be async-signal-safe on all OSes, and functions + /// called between `fork` and `exec` must be async-signal-safe. However, in practice, the + /// ability to `fork` then `chroot` then `exec` is a capability widely provided by Rust targets + /// and relied upon by numerous programs. + /// + /// `chroot` is not, by itself, a complete security or sandboxing measure. It may form some part + /// of a containment mechanism, but fully containing a process requires additional steps. + /// + /// See the [`chroot`] manual page for more details. + /// + /// [`chroot`]: https://man7.org/linux/man-pages/man2/chroot.2.html + /// + /// # Examples + /// + #[cfg_attr(target_family = "unix", doc = "```no_run")] + #[cfg_attr(not(target_family = "unix"), doc = "```ignore (needs unix)")] + /// #![feature(process_chroot)] + /// use std::process::Command; + /// use std::os::unix::process::CommandExt; + /// + /// Command::new("binary-within-new-root") + /// .chroot("/new-root") + /// .spawn()? + /// .wait()?; + /// # + /// # Ok::<_, Box>(()) + #[doc = "```"] #[unstable(feature = "process_chroot", issue = "141298")] fn chroot>(&mut self, dir: P) -> &mut process::Command; @@ -747,6 +777,15 @@ impl From for process::ChildStderr { } /// Returns the OS-assigned process identifier associated with this process's parent. +/// +/// # Example +/// +#[cfg_attr(target_family = "unix", doc = "```")] +#[cfg_attr(not(target_family = "unix"), doc = "```ignore (needs unix)")] +/// use std::os::unix::process; +/// let parent = process::parent_id(); +/// println!("The parent process has ID {}", parent); +/// ``` #[must_use] #[stable(feature = "unix_ppid", since = "1.27.0")] pub fn parent_id() -> u32 { diff --git a/library/std/src/os/windows/fs.rs b/library/std/src/os/windows/fs.rs index c10766470e873..865cc90e83156 100644 --- a/library/std/src/os/windows/fs.rs +++ b/library/std/src/os/windows/fs.rs @@ -69,9 +69,6 @@ pub trait FileExt { /// /// # Errors /// - /// If this function encounters an error of the kind - /// [`io::ErrorKind::Interrupted`] then the error is ignored and the operation - /// will continue. /// /// If this function encounters an "end of file" before completely filling /// the buffer, it returns an error of the kind [`io::ErrorKind::UnexpectedEof`]. @@ -113,7 +110,6 @@ pub trait FileExt { buf = &mut buf[n..]; offset += n as u64; } - Err(ref e) if e.is_interrupted() => {} Err(e) => return Err(e), } } @@ -203,7 +199,6 @@ pub trait FileExt { let prev_written = buf.written(); match self.seek_read_buf(buf.reborrow(), offset) { Ok(()) => {} - Err(e) if e.is_interrupted() => {} Err(e) => return Err(e), } let n = buf.written() - prev_written; @@ -264,8 +259,7 @@ pub trait FileExt { /// /// # Errors /// - /// This function will return the first error of - /// non-[`io::ErrorKind::Interrupted`] kind that [`seek_write`] returns. + /// This function will return the first error that [`seek_write`] returns. /// /// [`seek_write`]: FileExt::seek_write /// @@ -298,7 +292,6 @@ pub trait FileExt { buf = &buf[n..]; offset += n as u64 } - Err(ref e) if e.is_interrupted() => {} Err(e) => return Err(e), } } @@ -915,6 +908,7 @@ impl FileTimesExt for fs::FileTimes { /// the process as an administrator. /// /// [symlink-security]: https://docs.microsoft.com/en-us/windows/security/threat-protection/security-policy-settings/create-symbolic-links +#[doc(alias = "CreateSymbolicLink", alias = "CreateSymbolicLinkW")] #[stable(feature = "symlink", since = "1.1.0")] #[cfg_attr(not(test), rustc_diagnostic_item = "fs_symlink_file")] pub fn symlink_file, Q: AsRef>(original: P, link: Q) -> io::Result<()> { @@ -956,6 +950,7 @@ pub fn symlink_file, Q: AsRef>(original: P, link: Q) -> io: /// the process as an administrator. /// /// [symlink-security]: https://docs.microsoft.com/en-us/windows/security/threat-protection/security-policy-settings/create-symbolic-links +#[doc(alias = "CreateSymbolicLink", alias = "CreateSymbolicLinkW")] #[stable(feature = "symlink", since = "1.1.0")] #[cfg_attr(not(test), rustc_diagnostic_item = "fs_symlink_dir")] pub fn symlink_dir, Q: AsRef>(original: P, link: Q) -> io::Result<()> { diff --git a/library/std/src/os/windows/mod.rs b/library/std/src/os/windows/mod.rs index a7e032dbf4d4d..507d14765e3fa 100644 --- a/library/std/src/os/windows/mod.rs +++ b/library/std/src/os/windows/mod.rs @@ -36,6 +36,10 @@ pub mod process; pub mod raw; pub mod thread; +#[unstable(feature = "codeview_annotation", issue = "163964")] +#[cfg(windows)] +pub use core::os::windows::{CodeViewAnnotationArgs, codeview_annotation}; + /// A prelude for conveniently writing platform-specific code. /// /// Includes all extension traits, and some important type definitions. diff --git a/library/std/src/os/windows/net/addr.rs b/library/std/src/os/windows/net/addr.rs index c330432039a8f..5f6b93e56e723 100644 --- a/library/std/src/os/windows/net/addr.rs +++ b/library/std/src/os/windows/net/addr.rs @@ -1,5 +1,5 @@ #![unstable(feature = "windows_unix_domain_sockets", issue = "150487")] -use crate::bstr::ByteStr; +use crate::byte_str::ByteStr; use crate::ffi::OsStr; use crate::path::Path; #[cfg(not(doc))] diff --git a/library/std/src/os/windows/process.rs b/library/std/src/os/windows/process.rs index 971d88c65ea75..f4b60f2ed8eca 100644 --- a/library/std/src/os/windows/process.rs +++ b/library/std/src/os/windows/process.rs @@ -381,7 +381,10 @@ pub impl(self) trait CommandExt { /// see the [Remarks][1] section of the `CreateProcessW` documentation. /// /// [1]: https://learn.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-createprocessw#remarks - #[unstable(feature = "windows_process_extensions_inherit_handles", issue = "146407")] + #[stable( + feature = "windows_process_extensions_inherit_handles", + since = "CURRENT_RUSTC_VERSION" + )] fn inherit_handles(&mut self, inherit_handles: bool) -> &mut process::Command; } @@ -466,7 +469,7 @@ impl ChildExt for process::Child { } /// Windows-specific extensions to [`process::ExitCode`]. -#[unstable(feature = "windows_process_exit_code_from", issue = "111688")] +#[stable(feature = "windows_process_exit_code_from", since = "CURRENT_RUSTC_VERSION")] pub impl(self) trait ExitCodeExt { /// Creates a new `ExitCode` from the raw underlying `u32` return value of /// a process. @@ -474,11 +477,11 @@ pub impl(self) trait ExitCodeExt { /// The exit code should not be 259, as this conflicts with the `STILL_ACTIVE` /// macro returned from the `GetExitCodeProcess` function to signal that the /// process has yet to run to completion. - #[unstable(feature = "windows_process_exit_code_from", issue = "111688")] + #[stable(feature = "windows_process_exit_code_from", since = "CURRENT_RUSTC_VERSION")] fn from_raw(raw: u32) -> Self; } -#[unstable(feature = "windows_process_exit_code_from", issue = "111688")] +#[stable(feature = "windows_process_exit_code_from", since = "CURRENT_RUSTC_VERSION")] impl ExitCodeExt for process::ExitCode { fn from_raw(raw: u32) -> Self { process::ExitCode::from_inner(From::from(raw)) diff --git a/library/std/src/process.rs b/library/std/src/process.rs index 20ea03d36eb0e..feed6e1ad55a0 100644 --- a/library/std/src/process.rs +++ b/library/std/src/process.rs @@ -2676,7 +2676,7 @@ pub fn id() -> u32 { #[rustc_on_unimplemented(on( cause = "MainFunctionType", message = "`main` has invalid return type `{Self}`", - label = "`main` can only return types that implement `{This}`" + label = "`main` can only return types that implement `{this}`" ))] pub trait Termination { /// Is called to get the representation of the value as status code. diff --git a/library/std/src/sync/lazy_lock.rs b/library/std/src/sync/lazy_lock.rs index d1e5fd0af66f1..4462e8f7c193a 100644 --- a/library/std/src/sync/lazy_lock.rs +++ b/library/std/src/sync/lazy_lock.rs @@ -78,7 +78,7 @@ union Data { /// } /// ``` #[stable(feature = "lazy_cell", since = "1.80.0")] -pub struct LazyLock T> { +pub struct LazyLock<#[rustc_assert_variance(invariant)] T, F = fn() -> T> { // FIXME(nonpoison_once): if possible, switch to nonpoison version once it is available once: Once, data: UnsafeCell>, diff --git a/library/std/src/sync/nonpoison/mutex.rs b/library/std/src/sync/nonpoison/mutex.rs index a4ba86e38516c..c494234ddb357 100644 --- a/library/std/src/sync/nonpoison/mutex.rs +++ b/library/std/src/sync/nonpoison/mutex.rs @@ -47,7 +47,7 @@ use crate::sys::sync as sys; /// ``` #[unstable(feature = "nonpoison_mutex", issue = "134645")] #[cfg_attr(not(test), rustc_diagnostic_item = "NonPoisonMutex")] -pub struct Mutex { +pub struct Mutex<#[rustc_assert_variance(invariant)] T: ?Sized> { inner: sys::Mutex, data: UnsafeCell, } @@ -96,7 +96,10 @@ unsafe impl Sync for Mutex {} #[unstable(feature = "nonpoison_mutex", issue = "134645")] #[clippy::has_significant_drop] #[cfg_attr(not(test), rustc_diagnostic_item = "NonPoisonMutexGuard")] -pub struct MutexGuard<'a, T: ?Sized + 'a> { +pub struct MutexGuard< + #[rustc_assert_variance(covariant)] 'a, + #[rustc_assert_variance(invariant)] T: ?Sized + 'a, +> { lock: &'a Mutex, } @@ -138,7 +141,10 @@ unsafe impl Sync for MutexGuard<'_, T> {} #[unstable(feature = "mapped_lock_guards", issue = "117108")] // #[unstable(feature = "nonpoison_mutex", issue = "134645")] #[clippy::has_significant_drop] -pub struct MappedMutexGuard<'a, T: ?Sized + 'a> { +pub struct MappedMutexGuard< + #[rustc_assert_variance(covariant)] 'a, + #[rustc_assert_variance(invariant)] T: ?Sized + 'a, +> { // NB: we use a pointer instead of `&'a mut T` to avoid `noalias` violations, because a // `MappedMutexGuard` argument doesn't hold uniqueness for its whole scope, only until it drops. // `NonNull` is covariant over `T`, so we add a `PhantomData<&'a mut T>` field diff --git a/library/std/src/sync/nonpoison/rwlock.rs b/library/std/src/sync/nonpoison/rwlock.rs index 19064fdd1ce10..857d7c085b3e4 100644 --- a/library/std/src/sync/nonpoison/rwlock.rs +++ b/library/std/src/sync/nonpoison/rwlock.rs @@ -40,7 +40,7 @@ use crate::sys::sync as sys; /// ``` #[unstable(feature = "nonpoison_rwlock", issue = "134645")] #[cfg_attr(not(test), rustc_diagnostic_item = "NonPoisonRwLock")] -pub struct RwLock { +pub struct RwLock<#[rustc_assert_variance(invariant)] T: ?Sized> { /// The inner [`sys::RwLock`] that synchronizes thread access to the protected data. inner: sys::RwLock, /// The lock-protected data. @@ -72,7 +72,10 @@ unsafe impl Sync for RwLock {} #[unstable(feature = "nonpoison_rwlock", issue = "134645")] #[clippy::has_significant_drop] #[cfg_attr(not(test), rustc_diagnostic_item = "NonPoisonRwLockReadGuard")] -pub struct RwLockReadGuard<'rwlock, T: ?Sized + 'rwlock> { +pub struct RwLockReadGuard< + #[rustc_assert_variance(covariant)] 'rwlock, + #[rustc_assert_variance(covariant)] T: ?Sized + 'rwlock, +> { /// A pointer to the data protected by the `RwLock`. Note that we use a pointer here instead of /// `&'rwlock T` to avoid `noalias` violations, because a `RwLockReadGuard` instance only holds /// immutability until it drops, not for its whole scope. @@ -104,7 +107,10 @@ unsafe impl Sync for RwLockReadGuard<'_, T> {} #[unstable(feature = "nonpoison_rwlock", issue = "134645")] #[clippy::has_significant_drop] #[cfg_attr(not(test), rustc_diagnostic_item = "NonPoisonRwLockWriteGuard")] -pub struct RwLockWriteGuard<'rwlock, T: ?Sized + 'rwlock> { +pub struct RwLockWriteGuard< + #[rustc_assert_variance(covariant)] 'rwlock, + #[rustc_assert_variance(invariant)] T: ?Sized + 'rwlock, +> { /// A reference to the [`RwLock`] that we have write-locked. lock: &'rwlock RwLock, } @@ -130,7 +136,10 @@ unsafe impl Sync for RwLockWriteGuard<'_, T> {} #[unstable(feature = "mapped_lock_guards", issue = "117108")] // #[unstable(feature = "nonpoison_rwlock", issue = "134645")] #[clippy::has_significant_drop] -pub struct MappedRwLockReadGuard<'rwlock, T: ?Sized + 'rwlock> { +pub struct MappedRwLockReadGuard< + #[rustc_assert_variance(covariant)] 'rwlock, + #[rustc_assert_variance(covariant)] T: ?Sized + 'rwlock, +> { /// A pointer to the data protected by the `RwLock`. Note that we use a pointer here instead of /// `&'rwlock T` to avoid `noalias` violations, because a `MappedRwLockReadGuard` instance only /// holds immutability until it drops, not for its whole scope. @@ -164,7 +173,10 @@ unsafe impl Sync for MappedRwLockReadGuard<'_, T> {} #[unstable(feature = "mapped_lock_guards", issue = "117108")] // #[unstable(feature = "nonpoison_rwlock", issue = "134645")] #[clippy::has_significant_drop] -pub struct MappedRwLockWriteGuard<'rwlock, T: ?Sized + 'rwlock> { +pub struct MappedRwLockWriteGuard< + #[rustc_assert_variance(covariant)] 'rwlock, + #[rustc_assert_variance(invariant)] T: ?Sized + 'rwlock, +> { /// A pointer to the data protected by the `RwLock`. Note that we use a pointer here instead of /// `&'rwlock T` to avoid `noalias` violations, because a `MappedRwLockWriteGuard` instance only /// holds uniquneness until it drops, not for its whole scope. diff --git a/library/std/src/sync/once_lock.rs b/library/std/src/sync/once_lock.rs index e8e8f869dd038..3519adab66c4e 100644 --- a/library/std/src/sync/once_lock.rs +++ b/library/std/src/sync/once_lock.rs @@ -106,7 +106,7 @@ use crate::sync::Once; /// /// ``` #[stable(feature = "once_cell", since = "1.70.0")] -pub struct OnceLock { +pub struct OnceLock<#[rustc_assert_variance(invariant)] T> { // FIXME(nonpoison_once): switch to nonpoison version once it is available once: Once, // Whether or not the value is initialized is tracked by `once.is_completed()`. diff --git a/library/std/src/sync/oneshot.rs b/library/std/src/sync/oneshot.rs index c905b7363e2a5..4fb21bf1a714f 100644 --- a/library/std/src/sync/oneshot.rs +++ b/library/std/src/sync/oneshot.rs @@ -86,9 +86,8 @@ pub struct Sender { inner: mpmc::Sender, } -// SAFETY: Since the only methods in which synchronization must occur take full ownership of the -// [`Sender`], it is perfectly safe to share a `&Sender` between threads (as it is effectively -// useless without ownership). +// SAFETY: Sending a value requires ownership of the sender. Shared references only inspect +// channel state, so they cannot send a value of a non-`Send` type from another thread. #[unstable(feature = "oneshot_channel", issue = "143674")] unsafe impl Sync for Sender {} @@ -119,6 +118,20 @@ impl Sender { pub fn send(self, t: T) -> Result<(), SendError> { self.inner.send(t) } + + /// Returns `true` if the corresponding [`Receiver`] has been dropped. + /// + /// If this method returns `true`, [`send`](Self::send) is guaranteed to return an error. + /// + /// On the receiving side, [`Receiver::is_ready`] checks whether a value is available without + /// consuming the receiver. + /// + /// A return value of `false` does not guarantee that a later call to [`send`](Self::send) will + /// succeed. The receiver may be dropped immediately after this method returns. + #[unstable(feature = "oneshot_channel", issue = "143674")] + pub fn is_closed(&self) -> bool { + self.inner.is_disconnected() + } } #[unstable(feature = "oneshot_channel", issue = "143674")] diff --git a/library/std/src/sync/poison/mutex.rs b/library/std/src/sync/poison/mutex.rs index 42c5fcf3e27c0..2bb4af20140cd 100644 --- a/library/std/src/sync/poison/mutex.rs +++ b/library/std/src/sync/poison/mutex.rs @@ -224,7 +224,7 @@ use crate::sys::sync as sys; /// #[stable(feature = "rust1", since = "1.0.0")] #[cfg_attr(not(test), rustc_diagnostic_item = "Mutex")] -pub struct Mutex { +pub struct Mutex<#[rustc_assert_variance(invariant)] T: ?Sized> { inner: sys::Mutex, poison: poison::Flag, data: UnsafeCell, @@ -274,7 +274,10 @@ unsafe impl Sync for Mutex {} #[stable(feature = "rust1", since = "1.0.0")] #[clippy::has_significant_drop] #[cfg_attr(not(test), rustc_diagnostic_item = "MutexGuard")] -pub struct MutexGuard<'a, T: ?Sized + 'a> { +pub struct MutexGuard< + #[rustc_assert_variance(covariant)] 'a, + #[rustc_assert_variance(invariant)] T: ?Sized + 'a, +> { lock: &'a Mutex, poison: poison::Guard, } @@ -317,7 +320,10 @@ unsafe impl Sync for MutexGuard<'_, T> {} and cause Futures to not implement `Send`"] #[unstable(feature = "mapped_lock_guards", issue = "117108")] #[clippy::has_significant_drop] -pub struct MappedMutexGuard<'a, T: ?Sized + 'a> { +pub struct MappedMutexGuard< + #[rustc_assert_variance(covariant)] 'a, + #[rustc_assert_variance(invariant)] T: ?Sized + 'a, +> { // NB: we use a pointer instead of `&'a mut T` to avoid `noalias` violations, because a // `MappedMutexGuard` argument doesn't hold uniqueness for its whole scope, only until it drops. // `NonNull` is covariant over `T`, so we add a `PhantomData<&'a mut T>` field diff --git a/library/std/src/sync/poison/rwlock.rs b/library/std/src/sync/poison/rwlock.rs index de1fedf88f63a..686ac71d8227b 100644 --- a/library/std/src/sync/poison/rwlock.rs +++ b/library/std/src/sync/poison/rwlock.rs @@ -91,7 +91,7 @@ use crate::sys::sync as sys; /// [`Mutex`]: super::Mutex #[stable(feature = "rust1", since = "1.0.0")] #[cfg_attr(not(test), rustc_diagnostic_item = "RwLock")] -pub struct RwLock { +pub struct RwLock<#[rustc_assert_variance(invariant)] T: ?Sized> { /// The inner [`sys::RwLock`] that synchronizes thread access to the protected data. inner: sys::RwLock, /// A flag denoting if this `RwLock` has been poisoned. @@ -125,7 +125,10 @@ unsafe impl Sync for RwLock {} #[stable(feature = "rust1", since = "1.0.0")] #[clippy::has_significant_drop] #[cfg_attr(not(test), rustc_diagnostic_item = "RwLockReadGuard")] -pub struct RwLockReadGuard<'rwlock, T: ?Sized + 'rwlock> { +pub struct RwLockReadGuard< + #[rustc_assert_variance(covariant)] 'rwlock, + #[rustc_assert_variance(covariant)] T: ?Sized + 'rwlock, +> { /// A pointer to the data protected by the `RwLock`. Note that we use a pointer here instead of /// `&'rwlock T` to avoid `noalias` violations, because a `RwLockReadGuard` instance only holds /// immutability until it drops, not for its whole scope. @@ -157,7 +160,10 @@ unsafe impl Sync for RwLockReadGuard<'_, T> {} #[stable(feature = "rust1", since = "1.0.0")] #[clippy::has_significant_drop] #[cfg_attr(not(test), rustc_diagnostic_item = "RwLockWriteGuard")] -pub struct RwLockWriteGuard<'rwlock, T: ?Sized + 'rwlock> { +pub struct RwLockWriteGuard< + #[rustc_assert_variance(covariant)] 'rwlock, + #[rustc_assert_variance(invariant)] T: ?Sized + 'rwlock, +> { /// A reference to the [`RwLock`] that we have write-locked. lock: &'rwlock RwLock, /// The poison guard. See the [`poison`] module for more information. @@ -184,7 +190,10 @@ unsafe impl Sync for RwLockWriteGuard<'_, T> {} and cause Futures to not implement `Send`"] #[unstable(feature = "mapped_lock_guards", issue = "117108")] #[clippy::has_significant_drop] -pub struct MappedRwLockReadGuard<'rwlock, T: ?Sized + 'rwlock> { +pub struct MappedRwLockReadGuard< + #[rustc_assert_variance(covariant)] 'rwlock, + #[rustc_assert_variance(covariant)] T: ?Sized + 'rwlock, +> { /// A pointer to the data protected by the `RwLock`. Note that we use a pointer here instead of /// `&'rwlock T` to avoid `noalias` violations, because a `MappedRwLockReadGuard` instance only /// holds immutability until it drops, not for its whole scope. @@ -215,7 +224,10 @@ unsafe impl Sync for MappedRwLockReadGuard<'_, T> {} and cause Future's to not implement `Send`"] #[unstable(feature = "mapped_lock_guards", issue = "117108")] #[clippy::has_significant_drop] -pub struct MappedRwLockWriteGuard<'rwlock, T: ?Sized + 'rwlock> { +pub struct MappedRwLockWriteGuard< + #[rustc_assert_variance(covariant)] 'rwlock, + #[rustc_assert_variance(invariant)] T: ?Sized + 'rwlock, +> { /// A pointer to the data protected by the `RwLock`. Note that we use a pointer here instead of /// `&'rwlock T` to avoid `noalias` violations, because a `MappedRwLockWriteGuard` instance only /// holds uniquneness until it drops, not for its whole scope. diff --git a/library/std/src/sync/reentrant_lock.rs b/library/std/src/sync/reentrant_lock.rs index faa1bba1f5389..225bb19c7318d 100644 --- a/library/std/src/sync/reentrant_lock.rs +++ b/library/std/src/sync/reentrant_lock.rs @@ -78,7 +78,7 @@ use crate::thread::{ThreadId, current_id}; // we don't need to further synchronize the TID accesses, so they can be regular 64-bit // non-atomic accesses. #[unstable(feature = "reentrant_lock", issue = "121440")] -pub struct ReentrantLock { +pub struct ReentrantLock<#[rustc_assert_variance(covariant)] T: ?Sized> { mutex: sys::Mutex, owner: Tid, lock_count: UnsafeCell, @@ -204,7 +204,10 @@ impl RefUnwindSafe for ReentrantLock {} /// the guarded data. #[must_use = "if unused the ReentrantLock will immediately unlock"] #[unstable(feature = "reentrant_lock", issue = "121440")] -pub struct ReentrantLockGuard<'a, T: ?Sized + 'a> { +pub struct ReentrantLockGuard< + #[rustc_assert_variance(covariant)] 'a, + #[rustc_assert_variance(covariant)] T: ?Sized + 'a, +> { lock: &'a ReentrantLock, } diff --git a/library/std/src/sys/fs/hermit.rs b/library/std/src/sys/fs/hermit.rs index 89f6989b9adbe..32ed0bd76214f 100644 --- a/library/std/src/sys/fs/hermit.rs +++ b/library/std/src/sys/fs/hermit.rs @@ -221,6 +221,14 @@ impl FileType { } } +impl ReadDir { + pub fn dir(&self) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true); + Dir::open(&self.inner.root, &options) + } +} + impl fmt::Debug for ReadDir { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // This will only be called from std::fs::ReadDir, which will add a "ReadDir()" frame. @@ -594,6 +602,10 @@ pub fn rename(_old: &Path, _new: &Path) -> io::Result<()> { unsupported() } +pub fn rename_noreplace(_old: &Path, _new: &Path) -> io::Result<()> { + unsupported() +} + pub fn set_perm(_p: &Path, _perm: FilePermissions) -> io::Result<()> { unsupported() } diff --git a/library/std/src/sys/fs/mod.rs b/library/std/src/sys/fs/mod.rs index ca5c28dcd88bd..e15841b363d18 100644 --- a/library/std/src/sys/fs/mod.rs +++ b/library/std/src/sys/fs/mod.rs @@ -78,6 +78,10 @@ pub fn rename(old: &Path, new: &Path) -> io::Result<()> { with_native_path(old, &|old| with_native_path(new, &|new| imp::rename(old, new))) } +pub fn rename_noreplace(old: &Path, new: &Path) -> io::Result<()> { + with_native_path(old, &|old| with_native_path(new, &|new| imp::rename_noreplace(old, new))) +} + pub fn remove_dir(path: &Path) -> io::Result<()> { with_native_path(path, &imp::rmdir) } diff --git a/library/std/src/sys/fs/motor.rs b/library/std/src/sys/fs/motor.rs index 91cd90a6b0cb3..269ece5666494 100644 --- a/library/std/src/sys/fs/motor.rs +++ b/library/std/src/sys/fs/motor.rs @@ -324,6 +324,10 @@ pub fn rename(old: &Path, new: &Path) -> io::Result<()> { moto_rt::fs::rename(old, new).map_err(map_motor_error) } +pub fn rename_noreplace(_old: &Path, _new: &Path) -> io::Result<()> { + unsupported() +} + pub fn rmdir(path: &Path) -> io::Result<()> { let path = path.to_str().ok_or(io::Error::from(io::ErrorKind::InvalidFilename))?; moto_rt::fs::rmdir(path).map_err(map_motor_error) @@ -406,6 +410,14 @@ pub fn readdir(path: &Path) -> io::Result { }) } +impl ReadDir { + pub fn dir(&self) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true); + Dir::open(Path::new(&self.path), &options) + } +} + impl Iterator for ReadDir { type Item = io::Result; diff --git a/library/std/src/sys/fs/solid.rs b/library/std/src/sys/fs/solid.rs index dfdcacdcdf277..c43a4794e620e 100644 --- a/library/std/src/sys/fs/solid.rs +++ b/library/std/src/sys/fs/solid.rs @@ -159,6 +159,14 @@ pub fn readdir(p: &Path) -> io::Result { } } +impl ReadDir { + pub fn dir(&self) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true); + Dir::open(&self.inner.root, &options) + } +} + impl fmt::Debug for ReadDir { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // This will only be called from std::fs::ReadDir, which will add a "ReadDir()" frame. @@ -530,6 +538,10 @@ pub fn rename(old: &Path, new: &Path) -> io::Result<()> { Ok(()) } +pub fn rename_noreplace(_old: &Path, _new: &Path) -> io::Result<()> { + unsupported() +} + pub fn set_perm(p: &Path, perm: FilePermissions) -> io::Result<()> { // Solid does not support symlinks set_perm_nofollow(p, perm) diff --git a/library/std/src/sys/fs/uefi.rs b/library/std/src/sys/fs/uefi.rs index 5d0c219586e0e..945eaeaff1d7d 100644 --- a/library/std/src/sys/fs/uefi.rs +++ b/library/std/src/sys/fs/uefi.rs @@ -147,6 +147,14 @@ impl FileType { } } +impl ReadDir { + pub fn dir(&self) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true); + Dir::open(&self.0.path(), &options) + } +} + impl fmt::Debug for ReadDir { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let mut b = f.debug_struct("ReadDir"); @@ -474,6 +482,10 @@ pub fn rename(old: &Path, new: &Path) -> io::Result<()> { f.set_file_info(new_info) } +pub fn rename_noreplace(_old: &Path, _new: &Path) -> io::Result<()> { + unsupported() +} + pub fn set_perm(p: &Path, perm: FilePermissions) -> io::Result<()> { // UEFI does not support symlinks set_perm_nofollow(p, perm) diff --git a/library/std/src/sys/fs/unix.rs b/library/std/src/sys/fs/unix.rs index 0adc0b9136f63..f66fdeabb8a83 100644 --- a/library/std/src/sys/fs/unix.rs +++ b/library/std/src/sys/fs/unix.rs @@ -69,7 +69,7 @@ pub use crate::sys::fs::common::exists; pub use crate::sys::fs::common::{ExtraHomeDirs, ExtraMediaDirs}; use crate::sys::helpers::run_path_with_cstr; use crate::sys::time::SystemTime; -#[cfg(all(target_os = "linux", target_env = "gnu"))] +#[cfg(any(target_os = "linux", target_os = "android"))] use crate::sys::weak::syscall; #[cfg(target_os = "android")] use crate::sys::weak::weak; @@ -796,6 +796,39 @@ impl fmt::Debug for FilePermissions { } } +impl ReadDir { + pub fn dir(&self) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true); + + cfg_select! { + // Some targets don't have `dirfd`. Open `Dir` based on path. + any( + target_os = "redox", + target_os = "espidf", + target_os = "horizon", + target_os = "vita", + target_os = "nto", + target_os = "qnx", + target_os = "vxworks", + target_os = "l4re", + ) => Dir::open(&self.inner.root, &options), + // Use `dirfd` where possible. + _ => { + let fd = unsafe { libc::dirfd(self.inner.dirp.0) }; + // Make this FD into a directory handle. We don't actually drop it, + // so having an `OwnedFd` is fine. + let dir_handle = + mem::ManuallyDrop::new(dir::Dir(unsafe { OwnedFd::from_raw_fd(fd) })); + // We don't want to expose `fd` or even the underlying file description + // as the directory stream has state attached to it. So we open a completely + // new file description based on this one. + dir_handle.open_dir(Path::new("."), &options) + } + } + } +} + impl fmt::Debug for ReadDir { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // This will only be called from std::fs::ReadDir, which will add a "ReadDir()" frame. @@ -1309,6 +1342,7 @@ impl File { target_os = "cygwin", target_os = "illumos", target_os = "aix", + target_os = "haiku", target_os = "android", target_os = "redox", target_vendor = "apple", @@ -1332,6 +1366,7 @@ impl File { target_os = "cygwin", target_os = "illumos", target_os = "aix", + target_os = "haiku", target_os = "android", target_os = "redox", target_vendor = "apple", @@ -1355,6 +1390,7 @@ impl File { target_os = "cygwin", target_os = "illumos", target_os = "aix", + target_os = "haiku", target_os = "android", target_os = "redox", target_vendor = "apple", @@ -1390,6 +1426,7 @@ impl File { target_os = "cygwin", target_os = "illumos", target_os = "aix", + target_os = "haiku", target_os = "android", target_os = "redox", target_vendor = "apple", @@ -1425,6 +1462,7 @@ impl File { target_os = "cygwin", target_os = "illumos", target_os = "aix", + target_os = "haiku", target_os = "android", target_os = "redox", target_vendor = "apple", @@ -1862,6 +1900,59 @@ pub fn rename(old: &CStr, new: &CStr) -> io::Result<()> { cvt(unsafe { libc::rename(old.as_ptr(), new.as_ptr()) }).map(|_| ()) } +#[cfg(any(target_os = "linux", target_os = "android"))] +pub fn rename_noreplace(old: &CStr, new: &CStr) -> io::Result<()> { + syscall!( + fn renameat2( + olddirfd: c_int, + oldpath: *const libc::c_char, + newdirfd: c_int, + newpath: *const libc::c_char, + flags: libc::c_uint, + ) -> c_int; + ); + + // `RENAME_NOREPLACE` is `c_uint` on linux and `c_int` on Android, so casting is needed here. + let flags = libc::RENAME_NOREPLACE as libc::c_uint; + let res = cvt(unsafe { + renameat2(libc::AT_FDCWD, old.as_ptr(), libc::AT_FDCWD, new.as_ptr(), flags) + }); + + match res { + Ok(_) => Ok(()), + Err(err) => match err.raw_os_error() { + // The kernel predates `renameat2`. + Some(libc::ENOSYS) => link_then_unlink(old, new), + // The filesystem does not support `RENAME_NOREPLACE`. + // `EINVAL` also covers misuse, so report it unchanged unless the fallback found `new`. + Some(libc::EINVAL) => match link_then_unlink(old, new) { + Err(fallback) if fallback.raw_os_error() != Some(libc::EEXIST) => Err(err), + res => res, + }, + _ => Err(err), + }, + } +} + +#[cfg(target_vendor = "apple")] +pub fn rename_noreplace(old: &CStr, new: &CStr) -> io::Result<()> { + let res = cvt(unsafe { libc::renamex_np(old.as_ptr(), new.as_ptr(), libc::RENAME_EXCL) }); + match res { + Ok(_) => Ok(()), + Err(err) => match err.raw_os_error() { + // The filesystem does not support `RENAME_EXCL`. + // Unlike Linux, Darwin keeps `EINVAL` for misuse only. + Some(libc::ENOTSUP) => link_then_unlink(old, new), + _ => Err(err), + }, + } +} + +#[cfg(not(any(target_os = "linux", target_os = "android", target_vendor = "apple")))] +pub fn rename_noreplace(old: &CStr, new: &CStr) -> io::Result<()> { + link_then_unlink(old, new) +} + pub fn set_perm(p: &CStr, perm: FilePermissions) -> io::Result<()> { cvt_r(|| unsafe { libc::chmod(p.as_ptr(), perm.mode) }).map(|_| ()) } @@ -2062,6 +2153,11 @@ pub fn canonicalize(path: &CStr) -> io::Result { }))) } +fn link_then_unlink(old: &CStr, new: &CStr) -> io::Result<()> { + link(old, new)?; + unlink(old) +} + fn open_from(from: &Path) -> io::Result<(crate::fs::File, crate::fs::Metadata)> { use crate::fs::File; use crate::sys::fs::common::NOT_FILE_ERROR; @@ -2492,6 +2588,9 @@ mod remove_dir_impl { } } + /// `parent_fd` being `None` indicates that this is the root where we start. + /// In that case, `path` will only be deleted if it is a directory. + /// When it is `Some`, this will delete anything. fn remove_dir_all_recursive(parent_fd: Option, path: &CStr) -> io::Result<()> { // try opening as directory let fd = match openat_nofollow_dironly(parent_fd, path) { @@ -2503,7 +2602,8 @@ mod remove_dir_impl { Some(parent_fd) => { cvt(unsafe { unlinkat(parent_fd, path.as_ptr(), 0) }).map(drop) } - // ...unless this was supposed to be the deletion root directory + // ...unless this was supposed to be the deletion root directory. We don't want + // to delete a file at the root, so we just forward the error. None => Err(err), }; } @@ -2562,6 +2662,9 @@ mod remove_dir_impl { // into symlinks. let attr = lstat(p)?; if attr.file_type().is_symlink() { + // This would error in `remove_dir_all_recursive` as it is not a directory. + // We promise to delete directory symlinks, so use `unlink` to do that. On Unix, we + // inadvertently also delete symlinks to files. super::unlink(p) } else { remove_dir_all_recursive(None, p) diff --git a/library/std/src/sys/fs/unix/dir.rs b/library/std/src/sys/fs/unix/dir.rs index dd01db58c600d..2ac5695e3d72a 100644 --- a/library/std/src/sys/fs/unix/dir.rs +++ b/library/std/src/sys/fs/unix/dir.rs @@ -60,7 +60,7 @@ impl Dir { pub fn self_metadata(&self) -> io::Result { // Reuse the implementation for files, which should work for all FDs. let fd = self.0.as_raw_fd(); - let f = core::mem::ManuallyDrop::new(File( + let f = mem::ManuallyDrop::new(File( // SAFETY: we borrowed `self` so the FD will not be closed while this function runs. unsafe { FileDesc::from_raw_fd(fd) }, )); diff --git a/library/std/src/sys/fs/unsupported.rs b/library/std/src/sys/fs/unsupported.rs index 26677c7c7410a..78b4a1a698a33 100644 --- a/library/std/src/sys/fs/unsupported.rs +++ b/library/std/src/sys/fs/unsupported.rs @@ -29,6 +29,25 @@ pub struct FileType(!); #[derive(Debug)] pub struct DirBuilder {} +// Dummy Drops so that `needs_drop::()` etc. is true independent of platform, +// avoiding a portability hazard. +// The types not listed here are pure data and don’t implement Drop even when real. +impl Drop for File { + fn drop(&mut self) { + self.0 + } +} +impl Drop for ReadDir { + fn drop(&mut self) { + self.0 + } +} +impl Drop for DirEntry { + fn drop(&mut self) { + self.0 + } +} + impl FileAttr { pub fn size(&self) -> u64 { self.0 @@ -138,6 +157,12 @@ impl fmt::Debug for FileType { } } +impl ReadDir { + pub fn dir(&self) -> io::Result { + self.0 + } +} + impl fmt::Debug for ReadDir { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0 @@ -309,6 +334,10 @@ pub fn rename(_old: &Path, _new: &Path) -> io::Result<()> { unsupported() } +pub fn rename_noreplace(_old: &Path, _new: &Path) -> io::Result<()> { + unsupported() +} + pub fn set_perm(_p: &Path, perm: FilePermissions) -> io::Result<()> { match perm.0 {} } diff --git a/library/std/src/sys/fs/vexos.rs b/library/std/src/sys/fs/vexos.rs index 5043a4daa83b4..5dbcdfef7c80d 100644 --- a/library/std/src/sys/fs/vexos.rs +++ b/library/std/src/sys/fs/vexos.rs @@ -12,8 +12,8 @@ use crate::sys::{unsupported, unsupported_err}; #[path = "unsupported.rs"] mod unsupported_fs; pub use unsupported_fs::{ - Dir, DirBuilder, FileTimes, canonicalize, link, readlink, remove_dir_all, rename, rmdir, - symlink, unlink, + Dir, DirBuilder, FileTimes, canonicalize, link, readlink, remove_dir_all, rename, + rename_noreplace, rmdir, symlink, unlink, }; pub use crate::sys::fs::common::{ExtraHomeDirs, ExtraMediaDirs}; @@ -127,6 +127,12 @@ impl FileType { } } +impl ReadDir { + pub fn dir(&self) -> io::Result { + self.0 + } +} + impl fmt::Debug for ReadDir { fn fmt(&self, _f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0 @@ -479,6 +485,13 @@ impl Drop for File { } } +impl Drop for ReadDir { + fn drop(&mut self) { + // Dummy Drop so that `needs_drop::()` is true independent of platform. + self.0 + } +} + pub fn readdir(_p: &Path) -> io::Result { // While there *is* a userspace function for reading file directories, // the necessary implementation cannot currently be done cleanly, as diff --git a/library/std/src/sys/fs/windows.rs b/library/std/src/sys/fs/windows.rs index c4e0bb6ecf51f..c7dbc815b3434 100644 --- a/library/std/src/sys/fs/windows.rs +++ b/library/std/src/sys/fs/windows.rs @@ -113,6 +113,15 @@ impl fmt::Debug for c::FILETIME { #[derive(Debug)] pub struct DirBuilder; +impl ReadDir { + pub fn dir(&self) -> io::Result { + let mut options = OpenOptions::new(); + options.read(true); + // FIXME: use race-free handle-based approach instead. + Dir::open(&self.root, &options) + } +} + impl fmt::Debug for ReadDir { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { // This will only be called from std::fs::ReadDir, which will add a "ReadDir()" frame. @@ -1303,8 +1312,8 @@ pub fn unlink(path: &WCStr) -> io::Result<()> { } } -pub fn rename(old: &WCStr, new: &WCStr) -> io::Result<()> { - if unsafe { c::MoveFileExW(old.as_ptr(), new.as_ptr(), c::MOVEFILE_REPLACE_EXISTING) } == 0 { +fn rename_inner(old: &WCStr, new: &WCStr, move_flags: u32, rename_flags: u32) -> io::Result<()> { + if unsafe { c::MoveFileExW(old.as_ptr(), new.as_ptr(), move_flags) } == 0 { let err = api::get_last_error(); // if `MoveFileExW` fails with ERROR_ACCESS_DENIED then try to move // the file while ignoring the readonly attribute. @@ -1336,10 +1345,8 @@ pub fn rename(old: &WCStr, new: &WCStr) -> io::Result<()> { return Err(io::ErrorKind::OutOfMemory.into()); } - (&raw mut (*file_rename_info).Anonymous).write(c::FILE_RENAME_INFO_0 { - Flags: c::FILE_RENAME_FLAG_REPLACE_IF_EXISTS - | c::FILE_RENAME_FLAG_POSIX_SEMANTICS, - }); + (&raw mut (*file_rename_info).Anonymous) + .write(c::FILE_RENAME_INFO_0 { Flags: rename_flags }); (&raw mut (*file_rename_info).RootDirectory).write(ptr::null_mut()); // Don't include the NULL in the size @@ -1374,6 +1381,21 @@ pub fn rename(old: &WCStr, new: &WCStr) -> io::Result<()> { Ok(()) } +pub fn rename(old: &WCStr, new: &WCStr) -> io::Result<()> { + rename_inner( + old, + new, + c::MOVEFILE_REPLACE_EXISTING, + c::FILE_RENAME_FLAG_REPLACE_IF_EXISTS | c::FILE_RENAME_FLAG_POSIX_SEMANTICS, + ) +} + +pub fn rename_noreplace(old: &WCStr, new: &WCStr) -> io::Result<()> { + // Without `MOVEFILE_REPLACE_EXISTING`, `MoveFileExW` fails if the destination + // exists. + rename_inner(old, new, 0, c::FILE_RENAME_FLAG_POSIX_SEMANTICS) +} + pub fn rmdir(p: &WCStr) -> io::Result<()> { cvt(unsafe { c::RemoveDirectoryW(p.as_ptr()) })?; Ok(()) diff --git a/library/std/src/sys/io/is_terminal/windows.rs b/library/std/src/sys/io/is_terminal/windows.rs index b0c718d71f9f3..ec6667bffb355 100644 --- a/library/std/src/sys/io/is_terminal/windows.rs +++ b/library/std/src/sys/io/is_terminal/windows.rs @@ -12,10 +12,45 @@ fn handle_is_console(handle: BorrowedHandle<'_>) -> bool { return false; } - let mut out = 0; - if unsafe { c::GetConsoleMode(handle.as_raw_handle(), &mut out) != 0 } { - // False positives aren't possible. If we got a console then we definitely have a console. - return true; + cfg_select! { + target_vendor = "win7" => { + let mut out = 0; + if unsafe { c::GetConsoleMode(handle.as_raw_handle(), &mut out) != 0 } { + // False positives aren't possible. If we got a console then we definitely have a console. + return true; + } + + // Early return if the handle is not FILE_TYPE_PIPE, implying it cannot be an msys PTY. + if unsafe { c::GetFileType(handle.as_raw_handle()) != 0x0003 } { + return false; + } + } + _ => unsafe { + use crate::mem::MaybeUninit; + // `GetConsoleMode` incurs a remote RPC round-trip (= bad), so it's best avoided. + // `NtQueryVolumeInformationFile` lets us test the handle directly. + // The required `FILE_DEVICE_CONSOLE` type was introduced in Windows 8. + // Fun fact: `GetFileType` uses `NtQueryVolumeInformationFile` internally. + let mut io_status = MaybeUninit::::uninit(); + let mut device_info = MaybeUninit::::uninit(); + let status = c::NtQueryVolumeInformationFile( + handle.as_raw_handle(), + io_status.as_mut_ptr(), + device_info.as_mut_ptr().cast(), + size_of_val(&device_info) as u32, + c::FileFsDeviceInformation, + ); + if status < 0 { + return false; + } + + match device_info.assume_init_ref().DeviceType { + c::FILE_DEVICE_CONSOLE => return true, + // If it's a named pipe, it may be an msys PTY. + c::FILE_DEVICE_NAMED_PIPE => {} // aka FILE_TYPE_PIPE + _ => return false, + } + }, } // Otherwise, we fall back to an msys hack to see if we can detect the presence of a pty. @@ -23,11 +58,6 @@ fn handle_is_console(handle: BorrowedHandle<'_>) -> bool { } fn msys_tty_on(handle: BorrowedHandle<'_>) -> bool { - // Early return if the handle is not a pipe. - if unsafe { c::GetFileType(handle.as_raw_handle()) != c::FILE_TYPE_PIPE } { - return false; - } - /// Mirrors [`FILE_NAME_INFO`], giving it a fixed length that we can stack /// allocate /// diff --git a/library/std/src/sys/net/connection/sgx.rs b/library/std/src/sys/net/connection/sgx.rs index acebb8c5bd60c..8771a89568da8 100644 --- a/library/std/src/sys/net/connection/sgx.rs +++ b/library/std/src/sys/net/connection/sgx.rs @@ -9,6 +9,7 @@ use crate::sys::{AsInner, FromInner, IntoInner, TryIntoInner, sgx_ineffective, u use crate::time::Duration; const DEFAULT_FAKE_TTL: u32 = 64; +const DEFAULT_FAKE_HLIM: u8 = 64; #[derive(Debug, Clone)] pub struct Socket { @@ -243,6 +244,14 @@ impl TcpStream { sgx_ineffective(DEFAULT_FAKE_TTL) } + pub fn set_hop_limit_v6(&self, _: u8) -> io::Result<()> { + sgx_ineffective(()) + } + + pub fn hop_limit_v6(&self) -> io::Result { + sgx_ineffective(DEFAULT_FAKE_HLIM) + } + pub fn take_error(&self) -> io::Result> { Ok(None) } @@ -322,6 +331,14 @@ impl TcpListener { sgx_ineffective(DEFAULT_FAKE_TTL) } + pub fn set_hop_limit_v6(&self, _: u8) -> io::Result<()> { + sgx_ineffective(()) + } + + pub fn hop_limit_v6(&self) -> io::Result { + sgx_ineffective(DEFAULT_FAKE_HLIM) + } + pub fn set_only_v6(&self, _: bool) -> io::Result<()> { sgx_ineffective(()) } @@ -461,6 +478,22 @@ impl UdpSocket { self.0 } + pub fn set_hop_limit_v6(&self, _: u8) -> io::Result<()> { + self.0 + } + + pub fn hop_limit_v6(&self) -> io::Result { + self.0 + } + + pub fn set_multicast_hop_limit_v6(&self, _: u8) -> io::Result<()> { + self.0 + } + + pub fn multicast_hop_limit_v6(&self) -> io::Result { + self.0 + } + pub fn take_error(&self) -> io::Result> { self.0 } diff --git a/library/std/src/sys/net/connection/socket/mod.rs b/library/std/src/sys/net/connection/socket/mod.rs index 55db60fde694d..08375078b44ef 100644 --- a/library/std/src/sys/net/connection/socket/mod.rs +++ b/library/std/src/sys/net/connection/socket/mod.rs @@ -524,6 +524,31 @@ impl TcpStream { Ok(raw as u32) } + pub fn set_hop_limit_v6(&self, limit: u8) -> io::Result<()> { + cfg_select! { + all(target_os = "wasi", any(target_env = "p2", target_env = "p3")) => { + let _ = limit; + Err(io::Error::new(ErrorKind::Unsupported, "not supported on this platform")) + } + _ => unsafe { + setsockopt(&self.inner, c::IPPROTO_IPV6, c::IPV6_UNICAST_HOPS, limit as c_int) + }, + } + } + + pub fn hop_limit_v6(&self) -> io::Result { + cfg_select! { + all(target_os = "wasi", any(target_env = "p2", target_env = "p3")) => { + Err(io::Error::new(ErrorKind::Unsupported, "not supported on this platform")) + } + _ => { + let raw: c_int = + unsafe { getsockopt(&self.inner, c::IPPROTO_IPV6, c::IPV6_UNICAST_HOPS)? }; + Ok(raw as u8) + } + } + } + pub fn take_error(&self) -> io::Result> { self.inner.take_error() } @@ -652,6 +677,31 @@ impl TcpListener { Ok(raw as u32) } + pub fn set_hop_limit_v6(&self, limit: u8) -> io::Result<()> { + cfg_select! { + all(target_os = "wasi", any(target_env = "p2", target_env = "p3")) => { + let _ = limit; + Err(io::Error::new(ErrorKind::Unsupported, "not supported on this platform")) + } + _ => unsafe { + setsockopt(&self.inner, c::IPPROTO_IPV6, c::IPV6_UNICAST_HOPS, limit as c_int) + }, + } + } + + pub fn hop_limit_v6(&self) -> io::Result { + cfg_select! { + all(target_os = "wasi", any(target_env = "p2", target_env = "p3")) => { + Err(io::Error::new(ErrorKind::Unsupported, "not supported on this platform")) + } + _ => { + let raw: c_int = + unsafe { getsockopt(&self.inner, c::IPPROTO_IPV6, c::IPV6_UNICAST_HOPS)? }; + Ok(raw as u8) + } + } + } + pub fn set_only_v6(&self, only_v6: bool) -> io::Result<()> { unsafe { setsockopt(&self.inner, c::IPPROTO_IPV6, c::IPV6_V6ONLY, only_v6 as c_int) } } @@ -872,6 +922,56 @@ impl UdpSocket { Ok(raw as u32) } + pub fn set_hop_limit_v6(&self, limit: u8) -> io::Result<()> { + cfg_select! { + all(target_os = "wasi", any(target_env = "p2", target_env = "p3")) => { + let _ = limit; + Err(io::Error::new(ErrorKind::Unsupported, "not supported on this platform")) + } + _ => unsafe { + setsockopt(&self.inner, c::IPPROTO_IPV6, c::IPV6_UNICAST_HOPS, limit as c_int) + }, + } + } + + pub fn hop_limit_v6(&self) -> io::Result { + cfg_select! { + all(target_os = "wasi", any(target_env = "p2", target_env = "p3")) => { + Err(io::Error::new(ErrorKind::Unsupported, "not supported on this platform")) + } + _ => { + let raw: c_int = + unsafe { getsockopt(&self.inner, c::IPPROTO_IPV6, c::IPV6_UNICAST_HOPS)? }; + Ok(raw as u8) + } + } + } + + pub fn set_multicast_hop_limit_v6(&self, limit: u8) -> io::Result<()> { + cfg_select! { + all(target_os = "wasi", any(target_env = "p2", target_env = "p3")) => { + let _ = limit; + Err(io::Error::new(ErrorKind::Unsupported, "not supported on this platform")) + } + _ => unsafe { + setsockopt(&self.inner, c::IPPROTO_IPV6, c::IPV6_MULTICAST_HOPS, limit as c_int) + }, + } + } + + pub fn multicast_hop_limit_v6(&self) -> io::Result { + cfg_select! { + all(target_os = "wasi", any(target_env = "p2", target_env = "p3")) => { + Err(io::Error::new(ErrorKind::Unsupported, "not supported on this platform")) + } + _ => { + let raw: c_int = + unsafe { getsockopt(&self.inner, c::IPPROTO_IPV6, c::IPV6_MULTICAST_HOPS)? }; + Ok(raw as u8) + } + } + } + pub fn take_error(&self) -> io::Result> { self.inner.take_error() } diff --git a/library/std/src/sys/net/connection/socket/windows.rs b/library/std/src/sys/net/connection/socket/windows.rs index e9d5336001fd8..1249ac2808195 100644 --- a/library/std/src/sys/net/connection/socket/windows.rs +++ b/library/std/src/sys/net/connection/socket/windows.rs @@ -29,10 +29,11 @@ pub(super) mod netc { pub use crate::sys::c::{ ADDRESS_FAMILY as sa_family_t, ADDRINFOA as addrinfo, IP_ADD_MEMBERSHIP, IP_DROP_MEMBERSHIP, IP_MULTICAST_LOOP, IP_MULTICAST_TTL, IP_TTL, IPPROTO_IP, IPPROTO_IPV6, - IPV6_ADD_MEMBERSHIP, IPV6_DROP_MEMBERSHIP, IPV6_MULTICAST_LOOP, IPV6_V6ONLY, SO_BROADCAST, - SO_RCVTIMEO, SO_SNDTIMEO, SOCK_DGRAM, SOCK_STREAM, SOCKADDR as sockaddr, - SOCKADDR_STORAGE as sockaddr_storage, SOL_SOCKET, WSAEMSGSIZE as EMSGSIZE, bind, connect, - freeaddrinfo, getpeername, getsockname, getsockopt, listen, setsockopt, + IPV6_ADD_MEMBERSHIP, IPV6_DROP_MEMBERSHIP, IPV6_MULTICAST_HOPS, IPV6_MULTICAST_LOOP, + IPV6_UNICAST_HOPS, IPV6_V6ONLY, SO_BROADCAST, SO_RCVTIMEO, SO_SNDTIMEO, SOCK_DGRAM, + SOCK_STREAM, SOCKADDR as sockaddr, SOCKADDR_STORAGE as sockaddr_storage, SOL_SOCKET, + WSAEMSGSIZE as EMSGSIZE, bind, connect, freeaddrinfo, getpeername, getsockname, getsockopt, + listen, setsockopt, }; #[allow(non_camel_case_types)] diff --git a/library/std/src/sys/net/connection/uefi/mod.rs b/library/std/src/sys/net/connection/uefi/mod.rs index b29b324a1ee9e..0e72f87717568 100644 --- a/library/std/src/sys/net/connection/uefi/mod.rs +++ b/library/std/src/sys/net/connection/uefi/mod.rs @@ -145,6 +145,14 @@ impl TcpStream { self.inner.ttl() } + pub fn set_hop_limit_v6(&self, _: u8) -> io::Result<()> { + unsupported() + } + + pub fn hop_limit_v6(&self) -> io::Result { + unsupported() + } + pub fn take_error(&self) -> io::Result> { unsupported() } @@ -197,6 +205,14 @@ impl TcpListener { self.inner.ttl() } + pub fn set_hop_limit_v6(&self, _: u8) -> io::Result<()> { + unsupported() + } + + pub fn hop_limit_v6(&self) -> io::Result { + unsupported() + } + pub fn set_only_v6(&self, _: bool) -> io::Result<()> { unsupported() } @@ -324,6 +340,22 @@ impl UdpSocket { self.0 } + pub fn set_hop_limit_v6(&self, _: u8) -> io::Result<()> { + self.0 + } + + pub fn hop_limit_v6(&self) -> io::Result { + self.0 + } + + pub fn set_multicast_hop_limit_v6(&self, _: u8) -> io::Result<()> { + self.0 + } + + pub fn multicast_hop_limit_v6(&self) -> io::Result { + self.0 + } + pub fn take_error(&self) -> io::Result> { self.0 } diff --git a/library/std/src/sys/net/connection/unsupported.rs b/library/std/src/sys/net/connection/unsupported.rs index 7a4d3a97b8324..193348a79ae3d 100644 --- a/library/std/src/sys/net/connection/unsupported.rs +++ b/library/std/src/sys/net/connection/unsupported.rs @@ -111,6 +111,14 @@ impl TcpStream { self.0 } + pub fn set_hop_limit_v6(&self, _: u8) -> io::Result<()> { + self.0 + } + + pub fn hop_limit_v6(&self) -> io::Result { + self.0 + } + pub fn take_error(&self) -> io::Result> { self.0 } @@ -153,6 +161,14 @@ impl TcpListener { self.0 } + pub fn set_hop_limit_v6(&self, _: u8) -> io::Result<()> { + self.0 + } + + pub fn hop_limit_v6(&self) -> io::Result { + self.0 + } + pub fn set_only_v6(&self, _: bool) -> io::Result<()> { self.0 } @@ -279,6 +295,22 @@ impl UdpSocket { self.0 } + pub fn set_hop_limit_v6(&self, _: u8) -> io::Result<()> { + self.0 + } + + pub fn hop_limit_v6(&self) -> io::Result { + self.0 + } + + pub fn set_multicast_hop_limit_v6(&self, _: u8) -> io::Result<()> { + self.0 + } + + pub fn multicast_hop_limit_v6(&self) -> io::Result { + self.0 + } + pub fn take_error(&self) -> io::Result> { self.0 } diff --git a/library/std/src/sys/net/connection/wasip1.rs b/library/std/src/sys/net/connection/wasip1.rs index d461ea85f9fea..15d7f9a453141 100644 --- a/library/std/src/sys/net/connection/wasip1.rs +++ b/library/std/src/sys/net/connection/wasip1.rs @@ -169,6 +169,14 @@ impl TcpStream { unsupported() } + pub fn set_hop_limit_v6(&self, _: u8) -> io::Result<()> { + unsupported() + } + + pub fn hop_limit_v6(&self) -> io::Result { + unsupported() + } + pub fn take_error(&self) -> io::Result> { unsupported() } @@ -253,6 +261,14 @@ impl TcpListener { unsupported() } + pub fn set_hop_limit_v6(&self, _: u8) -> io::Result<()> { + unsupported() + } + + pub fn hop_limit_v6(&self) -> io::Result { + unsupported() + } + pub fn set_only_v6(&self, _: bool) -> io::Result<()> { unsupported() } @@ -425,6 +441,22 @@ impl UdpSocket { unsupported() } + pub fn set_hop_limit_v6(&self, _: u8) -> io::Result<()> { + unsupported() + } + + pub fn hop_limit_v6(&self) -> io::Result { + unsupported() + } + + pub fn set_multicast_hop_limit_v6(&self, _: u8) -> io::Result<()> { + unsupported() + } + + pub fn multicast_hop_limit_v6(&self) -> io::Result { + unsupported() + } + pub fn take_error(&self) -> io::Result> { unsupported() } diff --git a/library/std/src/sys/net/connection/xous/tcplistener.rs b/library/std/src/sys/net/connection/xous/tcplistener.rs index f406b4381f29a..e0730b2aac6c8 100644 --- a/library/std/src/sys/net/connection/xous/tcplistener.rs +++ b/library/std/src/sys/net/connection/xous/tcplistener.rs @@ -204,6 +204,14 @@ impl TcpListener { .map(|res| res[0] as _)?) } + pub fn set_hop_limit_v6(&self, hlim: u8) -> io::Result<()> { + unimpl!(); + } + + pub fn hop_limit_v6(&self) -> io::Result { + unimpl!(); + } + pub fn set_only_v6(&self, _: bool) -> io::Result<()> { unimpl!(); } diff --git a/library/std/src/sys/net/connection/xous/tcpstream.rs b/library/std/src/sys/net/connection/xous/tcpstream.rs index 45fe0418a27fa..47ff1c256c1b8 100644 --- a/library/std/src/sys/net/connection/xous/tcpstream.rs +++ b/library/std/src/sys/net/connection/xous/tcpstream.rs @@ -39,12 +39,13 @@ pub struct TcpStream { nonblocking: Arc>, } -fn sockaddr_to_buf(duration: Duration, addr: &SocketAddr, buf: &mut [u8]) { +fn sockaddr_to_buf(duration: Option, addr: &SocketAddr, buf: &mut [u8]) { // Construct the request. let port_bytes = addr.port().to_le_bytes(); buf[0] = port_bytes[0]; buf[1] = port_bytes[1]; - for (dest, src) in buf[2..].iter_mut().zip((duration.as_millis() as u64).to_le_bytes()) { + let millis = duration.map_or(0, |d| u64::try_from(d.as_millis()).unwrap_or(u64::MAX).max(1)); + for (dest, src) in buf[2..].iter_mut().zip(millis.to_le_bytes()) { *dest = src; } match addr.ip() { @@ -82,15 +83,11 @@ impl TcpStream { } } - pub fn connect(addr: A) -> io::Result { - each_addr(addr, |addr| Self::connect_timeout(addr, Duration::ZERO)) - } - - pub fn connect_timeout(addr: &SocketAddr, duration: Duration) -> io::Result { + fn connect_inner(addr: &SocketAddr, timeout: Option) -> io::Result { let mut connect_request = ConnectRequest { raw: [0u8; 4096] }; // Construct the request. - sockaddr_to_buf(duration, &addr, &mut connect_request.raw); + sockaddr_to_buf(timeout, &addr, &mut connect_request.raw); let Ok((_, valid)) = crate::os::xous::ffi::lend_mut( services::net_server(), @@ -138,6 +135,17 @@ impl TcpStream { }) } + pub fn connect(addr: A) -> io::Result { + each_addr(addr, |addr| Self::connect_inner(addr, None)) + } + + pub fn connect_timeout(addr: &SocketAddr, duration: Duration) -> io::Result { + if duration.is_zero() { + return Err(io::Error::ZERO_TIMEOUT); + } + Self::connect_inner(addr, Some(duration)) + } + pub fn set_read_timeout(&self, timeout: Option) -> io::Result<()> { if let Some(to) = timeout { if to.is_zero() { @@ -145,7 +153,7 @@ impl TcpStream { } } self.read_timeout.store( - timeout.map(|t| t.as_millis().min(u32::MAX as u128) as u32).unwrap_or_default(), + timeout.map_or(0, |t| u32::try_from(t.as_millis()).unwrap_or(u32::MAX).max(1)), Ordering::Relaxed, ); Ok(()) @@ -158,7 +166,7 @@ impl TcpStream { } } self.write_timeout.store( - timeout.map(|t| t.as_millis().min(u32::MAX as u128) as u32).unwrap_or_default(), + timeout.map_or(0, |t| u32::try_from(t.as_millis()).unwrap_or(u32::MAX).max(1)), Ordering::Relaxed, ); Ok(()) @@ -403,6 +411,14 @@ impl TcpStream { .map(|res| res[0] as _)?) } + pub fn set_hop_limit_v6(&self, _: u8) -> io::Result<()> { + unimpl!(); + } + + pub fn hop_limit_v6(&self) -> io::Result { + unimpl!(); + } + pub fn take_error(&self) -> io::Result> { // this call doesn't have a meaning on our platform, but we can at least not panic if it's used. Ok(None) diff --git a/library/std/src/sys/net/connection/xous/udp.rs b/library/std/src/sys/net/connection/xous/udp.rs index 281639488a015..e254bfe1a5e45 100644 --- a/library/std/src/sys/net/connection/xous/udp.rs +++ b/library/std/src/sys/net/connection/xous/udp.rs @@ -385,6 +385,14 @@ impl UdpSocket { .map(|res| res[0] as _)?) } + pub fn set_hop_limit_v6(&self, _: u8) -> io::Result<()> { + unimpl!(); + } + + pub fn hop_limit_v6(&self) -> io::Result { + unimpl!(); + } + pub fn take_error(&self) -> io::Result> { // this call doesn't have a meaning on our platform, but we can at least not panic if it's used. Ok(None) @@ -420,6 +428,14 @@ impl UdpSocket { unimpl!(); } + pub fn set_multicast_hop_limit_v6(&self, _: u8) -> io::Result<()> { + unimpl!(); + } + + pub fn multicast_hop_limit_v6(&self) -> io::Result { + unimpl!(); + } + pub fn set_multicast_loop_v6(&self, _: bool) -> io::Result<()> { unimpl!(); } diff --git a/library/std/src/sys/os_str/bytes.rs b/library/std/src/sys/os_str/bytes.rs index 048e5def1369b..040a9568ca1f6 100644 --- a/library/std/src/sys/os_str/bytes.rs +++ b/library/std/src/sys/os_str/bytes.rs @@ -4,7 +4,7 @@ use core::clone::CloneToUninit; use crate::borrow::Cow; -use crate::bstr::ByteStr; +use crate::byte_str::ByteStr; use crate::collections::TryReserveError; use crate::rc::Rc; use crate::sync::Arc; diff --git a/library/std/src/sys/pal/solid/abi/sockets.rs b/library/std/src/sys/pal/solid/abi/sockets.rs index 80802dd42e248..bedcf1c8e1e76 100644 --- a/library/std/src/sys/pal/solid/abi/sockets.rs +++ b/library/std/src/sys/pal/solid/abi/sockets.rs @@ -119,8 +119,11 @@ pub struct sockaddr_storage { pub ss_family: sa_family_t, pub s2_data1: [c_char; 2usize], pub s2_data2: [u32; 3usize], + pub s2_data3: [u32; 3usize], } +const _: () = assert!(size_of::() >= size_of::()); + #[repr(C)] #[derive(Debug, Copy, Clone)] pub struct addrinfo { diff --git a/library/std/src/sys/pal/windows/c.rs b/library/std/src/sys/pal/windows/c.rs index 25031968a7e02..439edba0871d5 100644 --- a/library/std/src/sys/pal/windows/c.rs +++ b/library/std/src/sys/pal/windows/c.rs @@ -20,6 +20,11 @@ pub type WCHAR = u16; pub const INVALID_HANDLE_VALUE: HANDLE = ::core::ptr::without_provenance_mut(-1i32 as _); +#[cfg(not(target_vendor = "win7"))] +pub const FILE_DEVICE_NAMED_PIPE: u32 = 0x00000011; +#[cfg(not(target_vendor = "win7"))] +pub const FILE_DEVICE_CONSOLE: u32 = 0x00000050; + // https://learn.microsoft.com/en-us/cpp/c-runtime-library/exit-success-exit-failure?view=msvc-170 pub const EXIT_SUCCESS: u32 = 0; pub const EXIT_FAILURE: u32 = 1; @@ -144,12 +149,6 @@ pub const HANDLE_FLAG_INHERIT: u32 = cu(windows_sys::HANDLE_FLAG_INHERIT); pub const STACK_SIZE_PARAM_IS_A_RESERVATION: u32 = cu(windows_sys::STACK_SIZE_PARAM_IS_A_RESERVATION); -// dwFlags -- WideCharToMultiByte -pub const WC_ERR_INVALID_CHARS: u32 = cu(windows_sys::WC_ERR_INVALID_CHARS); - -// dwFlags -- MultiByteToWideChar -pub const MB_ERR_INVALID_CHARS: u32 = cu(windows_sys::MB_ERR_INVALID_CHARS); - // dwFlags -- CreateWaitableTimerExW pub const CREATE_WAITABLE_TIMER_HIGH_RESOLUTION: u32 = cu(windows_sys::CREATE_WAITABLE_TIMER_HIGH_RESOLUTION); @@ -188,9 +187,6 @@ pub const FILE_CURRENT: u32 = cu(windows_sys::FILE_CURRENT); pub const FSCTL_GET_REPARSE_POINT: u32 = cu(windows_sys::FSCTL_GET_REPARSE_POINT); pub const FSCTL_SET_REPARSE_POINT: u32 = cu(windows_sys::FSCTL_SET_REPARSE_POINT); -// GetFileType return value (DWORD) -pub const FILE_TYPE_PIPE: u32 = cu(windows_sys::FILE_TYPE_PIPE); - // GetACP return value (UINT) pub const CP_UTF8: u32 = windows_sys::CP_UTF8.cast_unsigned(); @@ -473,6 +469,7 @@ cfg_select! { target_vendor = "uwp" => { windows_link::link_raw_dylib!("ntdll.dll" "system" fn NtCreateFile(filehandle : *mut HANDLE, desiredaccess : ACCESS_MASK, objectattributes : *const OBJECT_ATTRIBUTES, iostatusblock : *mut IO_STATUS_BLOCK, allocationsize : *const i64, fileattributes : u32, shareaccess : u32, createdisposition : u32, createoptions : u32, eabuffer : *const core::ffi::c_void, ealength : u32) -> NTSTATUS); windows_link::link_raw_dylib!("ntdll.dll" "system" fn NtOpenFile(filehandle : *mut HANDLE, desiredaccess : ACCESS_MASK, objectattributes : *const OBJECT_ATTRIBUTES, iostatusblock : *mut IO_STATUS_BLOCK, shareaccess : u32, openoptions : u32) -> NTSTATUS); + windows_link::link_raw_dylib!("ntdll.dll" "system" fn NtQueryVolumeInformationFile(filehandle : HANDLE, iostatusblock : *mut IO_STATUS_BLOCK, fsinformation : *mut core::ffi::c_void, length : u32, fsinformationclass : FS_INFORMATION_CLASS) -> NTSTATUS); windows_link::link_raw_dylib!("ntdll.dll" "system" fn NtReadFile(filehandle : HANDLE, event : HANDLE, apcroutine : PIO_APC_ROUTINE, apccontext : *const core::ffi::c_void, iostatusblock : *mut IO_STATUS_BLOCK, buffer : *mut core::ffi::c_void, length : u32, byteoffset : *const i64, key : *const u32) -> NTSTATUS); windows_link::link_raw_dylib!("ntdll.dll" "system" fn NtWriteFile(filehandle : HANDLE, event : HANDLE, apcroutine : PIO_APC_ROUTINE, apccontext : *const core::ffi::c_void, iostatusblock : *mut IO_STATUS_BLOCK, buffer : *const core::ffi::c_void, length : u32, byteoffset : *const i64, key : *const u32) -> NTSTATUS); windows_link::link_raw_dylib!("ntdll.dll" "system" fn RtlNtStatusToDosError(status : NTSTATUS) -> u32); diff --git a/library/std/src/sys/pal/windows/c/bindings.txt b/library/std/src/sys/pal/windows/c/bindings.txt index 70af36f27907e..1c7035028e83f 100644 --- a/library/std/src/sys/pal/windows/c/bindings.txt +++ b/library/std/src/sys/pal/windows/c/bindings.txt @@ -2032,6 +2032,7 @@ FILE_FLAG_RANDOM_ACCESS FILE_FLAG_SEQUENTIAL_SCAN FILE_FLAG_SESSION_AWARE FILE_FLAG_WRITE_THROUGH +FILE_FS_DEVICE_INFORMATION FILE_GENERIC_EXECUTE FILE_GENERIC_READ FILE_GENERIC_WRITE @@ -2084,7 +2085,6 @@ FILE_SYNCHRONOUS_IO_NONALERT FILE_TRAVERSE FILE_TYPE_CHAR FILE_TYPE_DISK -FILE_TYPE_PIPE FILE_TYPE_REMOTE FILE_TYPE_UNKNOWN FILE_WRITE_ATTRIBUTES @@ -2100,6 +2100,7 @@ FileCompressionInfo FileDispositionInfo FileDispositionInfoEx FileEndOfFileInfo +FileFsDeviceInformation FileFullDirectoryInfo FileFullDirectoryRestartInfo FileIdBothDirectoryInfo @@ -2259,7 +2260,9 @@ IPPROTO_UDP IPV6_ADD_MEMBERSHIP IPV6_DROP_MEMBERSHIP IPV6_MREQ +IPV6_MULTICAST_HOPS IPV6_MULTICAST_LOOP +IPV6_UNICAST_HOPS IPV6_V6ONLY IsThreadAFiber KF_FLAG_DONT_VERIFY @@ -2283,7 +2286,6 @@ MAX_PATH MAXIMUM_REPARSE_DATA_BUFFER_SIZE MaximumFileInfoByHandleClass MB_COMPOSITE -MB_ERR_INVALID_CHARS MB_PRECOMPOSED MB_USEGLYPHCHARS MOVEFILE_COPY_ALLOWED @@ -2303,6 +2305,7 @@ NO_ERROR NORMAL_PRIORITY_CLASS NtCreateFile NtOpenFile +NtQueryVolumeInformationFile NtReadFile NtSetInformationFile NtWriteFile @@ -2498,7 +2501,6 @@ WakeAllConditionVariable WakeByAddressAll WakeByAddressSingle WakeConditionVariable -WC_ERR_INVALID_CHARS WideCharToMultiByte WIN32_FIND_DATAW WRITE_DAC diff --git a/library/std/src/sys/pal/windows/c/windows_sys.rs b/library/std/src/sys/pal/windows/c/windows_sys.rs index 0c865494a6e7c..131d410edc9ad 100644 --- a/library/std/src/sys/pal/windows/c/windows_sys.rs +++ b/library/std/src/sys/pal/windows/c/windows_sys.rs @@ -78,6 +78,7 @@ windows_link::link!("kernel32.dll" "system" fn MoveFileExW(lpexistingfilename : windows_link::link!("kernel32.dll" "system" fn MultiByteToWideChar(codepage : u32, dwflags : u32, lpmultibytestr : *const i8, cbmultibyte : i32, lpwidecharstr : PWSTR, cchwidechar : i32) -> i32); windows_link::link!("ntdll.dll" "system" fn NtCreateFile(filehandle : *mut HANDLE, desiredaccess : ACCESS_MASK, objectattributes : *const OBJECT_ATTRIBUTES, iostatusblock : *mut IO_STATUS_BLOCK, allocationsize : *const i64, fileattributes : u32, shareaccess : u32, createdisposition : u32, createoptions : u32, eabuffer : *const core::ffi::c_void, ealength : u32) -> NTSTATUS); windows_link::link!("ntdll.dll" "system" fn NtOpenFile(filehandle : *mut HANDLE, desiredaccess : ACCESS_MASK, objectattributes : *const OBJECT_ATTRIBUTES, iostatusblock : *mut IO_STATUS_BLOCK, shareaccess : u32, openoptions : u32) -> NTSTATUS); +windows_link::link!("ntdll.dll" "system" fn NtQueryVolumeInformationFile(filehandle : HANDLE, iostatusblock : *mut IO_STATUS_BLOCK, fsinformation : *mut core::ffi::c_void, length : u32, fsinformationclass : FS_INFORMATION_CLASS) -> NTSTATUS); windows_link::link!("ntdll.dll" "system" fn NtReadFile(filehandle : HANDLE, event : HANDLE, apcroutine : PIO_APC_ROUTINE, apccontext : *const core::ffi::c_void, iostatusblock : *mut IO_STATUS_BLOCK, buffer : *mut core::ffi::c_void, length : u32, byteoffset : *const i64, key : *const u32) -> NTSTATUS); windows_link::link!("ntdll.dll" "system" fn NtSetInformationFile(filehandle : HANDLE, iostatusblock : *mut IO_STATUS_BLOCK, fileinformation : *const core::ffi::c_void, length : u32, fileinformationclass : FILE_INFORMATION_CLASS) -> NTSTATUS); windows_link::link!("ntdll.dll" "system" fn NtWriteFile(filehandle : HANDLE, event : HANDLE, apcroutine : PIO_APC_ROUTINE, apccontext : *const core::ffi::c_void, iostatusblock : *mut IO_STATUS_BLOCK, buffer : *const core::ffi::c_void, length : u32, byteoffset : *const i64, key : *const u32) -> NTSTATUS); @@ -2544,6 +2545,12 @@ pub const FILE_FLAG_RANDOM_ACCESS: i32 = 268435456; pub const FILE_FLAG_SEQUENTIAL_SCAN: i32 = 134217728; pub const FILE_FLAG_SESSION_AWARE: i32 = 8388608; pub const FILE_FLAG_WRITE_THROUGH: u32 = 2147483648; +#[repr(C)] +#[derive(Clone, Copy, Default)] +pub struct FILE_FS_DEVICE_INFORMATION { + pub DeviceType: u32, + pub Characteristics: u32, +} pub const FILE_GENERIC_EXECUTE: i32 = 1179808; pub const FILE_GENERIC_READ: i32 = 1179785; pub const FILE_GENERIC_WRITE: i32 = 1179926; @@ -2678,7 +2685,6 @@ pub const FILE_SYNCHRONOUS_IO_NONALERT: i32 = 32; pub const FILE_TRAVERSE: i32 = 32; pub const FILE_TYPE_CHAR: i32 = 2; pub const FILE_TYPE_DISK: i32 = 1; -pub const FILE_TYPE_PIPE: i32 = 3; pub const FILE_TYPE_REMOTE: i32 = 32768; pub const FILE_TYPE_UNKNOWN: i32 = 0; pub const FILE_WRITE_ATTRIBUTES: i32 = 256; @@ -2718,6 +2724,7 @@ pub const FORMAT_MESSAGE_IGNORE_INSERTS: i32 = 512; pub const FRS_ERR_SYSVOL_POPULATE_TIMEOUT: i32 = 8014; pub const FSCTL_GET_REPARSE_POINT: i32 = 589992; pub const FSCTL_SET_REPARSE_POINT: i32 = 589988; +pub type FS_INFORMATION_CLASS = i32; pub const FileAlignmentInfo: FILE_INFO_BY_HANDLE_CLASS = 17; pub const FileAllocationInfo: FILE_INFO_BY_HANDLE_CLASS = 5; pub const FileAttributeTagInfo: FILE_INFO_BY_HANDLE_CLASS = 9; @@ -2727,6 +2734,7 @@ pub const FileCompressionInfo: FILE_INFO_BY_HANDLE_CLASS = 8; pub const FileDispositionInfo: FILE_INFO_BY_HANDLE_CLASS = 4; pub const FileDispositionInfoEx: FILE_INFO_BY_HANDLE_CLASS = 21; pub const FileEndOfFileInfo: FILE_INFO_BY_HANDLE_CLASS = 6; +pub const FileFsDeviceInformation: FS_INFORMATION_CLASS = 4; pub const FileFullDirectoryInfo: FILE_INFO_BY_HANDLE_CLASS = 14; pub const FileFullDirectoryRestartInfo: FILE_INFO_BY_HANDLE_CLASS = 15; pub const FileIdBothDirectoryInfo: FILE_INFO_BY_HANDLE_CLASS = 10; @@ -2910,7 +2918,9 @@ impl Default for IPV6_MREQ { unsafe { core::mem::zeroed() } } } +pub const IPV6_MULTICAST_HOPS: i32 = 10; pub const IPV6_MULTICAST_LOOP: i32 = 11; +pub const IPV6_UNICAST_HOPS: i32 = 4; pub const IPV6_V6ONLY: i32 = 27; pub const IP_ADD_MEMBERSHIP: i32 = 12; pub const IP_DROP_MEMBERSHIP: i32 = 13; @@ -2977,7 +2987,6 @@ pub struct M128A { pub const MAXIMUM_REPARSE_DATA_BUFFER_SIZE: i32 = 16384; pub const MAX_PATH: i32 = 260; pub const MB_COMPOSITE: i32 = 2; -pub const MB_ERR_INVALID_CHARS: i32 = 8; pub const MB_PRECOMPOSED: i32 = 1; pub const MB_USEGLYPHCHARS: i32 = 4; pub const MOVEFILE_COPY_ALLOWED: i32 = 2; @@ -3343,7 +3352,6 @@ pub const WAIT_FAILED: u32 = 4294967295; pub const WAIT_IO_COMPLETION: i32 = 192; pub const WAIT_OBJECT_0: i32 = 0; pub const WAIT_TIMEOUT: i32 = 258; -pub const WC_ERR_INVALID_CHARS: i32 = 128; #[repr(C)] #[derive(Clone, Copy)] pub struct WIN32_FIND_DATAW { diff --git a/library/std/src/sys/platform_version/darwin/mod.rs b/library/std/src/sys/platform_version/darwin/mod.rs index 4756716452c88..7e82c04fce76c 100644 --- a/library/std/src/sys/platform_version/darwin/mod.rs +++ b/library/std/src/sys/platform_version/darwin/mod.rs @@ -3,7 +3,7 @@ use self::core_foundation::{ kCFPropertyListImmutable, kCFStringEncodingUTF8, }; use crate::borrow::Cow; -use crate::bstr::ByteStr; +use crate::byte_str::ByteStr; use crate::ffi::{CStr, c_char}; use crate::num::{NonZero, ParseIntError}; use crate::path::{Path, PathBuf}; diff --git a/library/std/src/sys/stdio/windows.rs b/library/std/src/sys/stdio/windows.rs index d9c9290ed21a2..f669527352246 100644 --- a/library/std/src/sys/stdio/windows.rs +++ b/library/std/src/sys/stdio/windows.rs @@ -1,14 +1,15 @@ #![unstable(issue = "none", feature = "windows_stdio")] -use core::ffi::c_void; use core::str::utf8_char_width; +#[cfg(not(target_vendor = "win7"))] +use crate::alloc::System; use crate::mem::MaybeUninit; use crate::os::windows::io::{FromRawHandle, IntoRawHandle}; use crate::sys::handle::Handle; use crate::sys::pal::api::{self, WinError}; use crate::sys::{c, cvt}; -use crate::{cmp, io, ptr, str}; +use crate::{cmp, io, ptr}; #[cfg(test)] mod tests; @@ -22,35 +23,12 @@ pub struct Stdin { pub struct Stdout { incomplete_utf8: IncompleteUtf8, - write_mode: Option, } pub struct Stderr { incomplete_utf8: IncompleteUtf8, - write_mode: Option, -} - -/// How to write to a standard stream, cached for the duration of a lock -/// session (see `Stdout::refresh`). Re-querying this on every lock -/// acquisition keeps `SetStdHandle` calls made between lock sessions working -/// (see #40490), while a held lock skips the per-write console queries that -/// otherwise dominate bulk writes (see #154071). -#[derive(Clone, Copy)] -enum WriteMode { - /// The stream is a pipe or file, or a console using the UTF-8 code page: - /// bytes are written out unchanged. - Passthrough(c::HANDLE), - /// The stream is a console using a non-UTF-8 code page: data is converted - /// to UTF-16 and written with `WriteConsoleW`. - Utf16Console(c::HANDLE), } -// SAFETY: a `HANDLE` stored here is only an OS handle value for a standard -// stream (it is never dereferenced), and handle values are valid -// process-wide, on any thread. -unsafe impl Send for WriteMode {} -unsafe impl Sync for WriteMode {} - struct IncompleteUtf8 { bytes: [u8; 4], len: u8, @@ -75,13 +53,13 @@ impl IncompleteUtf8 { } } -// Apparently Windows doesn't handle large reads on stdin or writes to stdout/stderr well (see -// #13304 for details). -// -// From MSDN (2011): "The storage for this buffer is allocated from a shared heap for the -// process that is 64 KB in size. The maximum size of the buffer will depend on heap usage." +// The way the console used to work up to Windows 8 is that the "LPC"s carried +// a small scratch space for reads/writes. If the client exceeded it, the server +// allocated a larger buffer from a fixed size section object. Its size was +// `CONSOLE_PORT_MEMORY_SIZE`, aka 64KiB. This meant any IO larger than that failed. +// If the Windows 7 source code ever leaks: The relevant function is `ConsoleAllocateCaptureBuffer`. // -// We choose the cap at 8 KiB because libuv does the same, and it seems to be acceptable so far. +// It remains a good size for a stack allocated buffer, however. const MAX_BUFFER_SIZE: usize = 8192; // The standard buffer size of BufReader for Stdin should be able to hold 3x more bytes than there @@ -89,6 +67,9 @@ const MAX_BUFFER_SIZE: usize = 8192; // UTF-16 to UTF-8. pub const STDIN_BUF_SIZE: usize = MAX_BUFFER_SIZE / 2 * 3; +#[cfg(not(target_vendor = "win7"))] +const MAX_WRITE_HEAP_LEN: usize = 64 * 1024; // In u16 -> 128KiB + pub fn get_handle(handle_id: u32) -> io::Result { let handle = unsafe { c::GetStdHandle(handle_id) }; if handle == c::INVALID_HANDLE_VALUE { @@ -100,62 +81,55 @@ pub fn get_handle(handle_id: u32) -> io::Result { } } +/// Returns true if `handle` refers to a proper console (aka conhost). fn is_console(handle: c::HANDLE) -> bool { - // `GetConsoleMode` will return false (0) if this is a pipe (we don't care about the reported - // mode). This will only detect Windows Console, not other terminals connected to a pipe like - // MSYS. Which is exactly what we need, as only Windows Console needs a conversion to UTF-16. - let mut mode = 0; - unsafe { c::GetConsoleMode(handle, &mut mode) != 0 } -} - -/// Returns true if the attached console's code page is currently UTF-8. -#[cfg(not(target_vendor = "win7"))] -fn is_utf8_console() -> bool { - unsafe { c::GetConsoleOutputCP() == c::CP_UTF8 } -} - -#[cfg(target_vendor = "win7")] -fn is_utf8_console() -> bool { - // Windows 7 has a fun "feature" where WriteFile on a console handle will return - // the number of UTF-16 code units written and not the number of bytes from the input string. - // So we always claim the console isn't UTF-8 to trigger the WriteConsole fallback code. - false + cfg_select! { + target_vendor = "win7" => { + // `GetConsoleMode` will return false (0) if this is a pipe (we don't care about the reported + // mode). This will only detect Windows Console, not other terminals connected to a pipe like + // MSYS. Which is exactly what we need, as only Windows Console needs a conversion to UTF-16. + let mut mode = 0; + unsafe { c::GetConsoleMode(handle, &mut mode) != 0 } + } + _ => unsafe { + // `GetConsoleMode` incurs a remote RPC round-trip (= bad), so it's best avoided. + // `NtQueryVolumeInformationFile` lets us test the handle directly. + // The required `FILE_DEVICE_CONSOLE` type was introduced in Windows 8. + // Fun fact: `GetFileType` uses `NtQueryVolumeInformationFile` internally. + let mut io_status = MaybeUninit::::uninit(); + let mut device_info = MaybeUninit::::uninit(); + let status = c::NtQueryVolumeInformationFile( + handle, + io_status.as_mut_ptr(), + device_info.as_mut_ptr().cast(), + size_of_val(&device_info) as u32, + c::FileFsDeviceInformation, + ); + status >= 0 && device_info.assume_init_ref().DeviceType == c::FILE_DEVICE_CONSOLE + }, + } } -fn write( - handle_id: u32, - data: &[u8], - incomplete_utf8: &mut IncompleteUtf8, - write_mode: &mut Option, -) -> io::Result { +fn write(handle_id: u32, data: &[u8], incomplete_utf8: &mut IncompleteUtf8) -> io::Result { if data.is_empty() { return Ok(0); } - let mode = match *write_mode { - Some(mode) => mode, - None => { - let handle = get_handle(handle_id)?; - let mode = if !is_console(handle) || is_utf8_console() { - WriteMode::Passthrough(handle) - } else { - WriteMode::Utf16Console(handle) - }; - // Only cache success; if there is no handle, keep erroring on - // every write like before. - *write_mode = Some(mode); - mode - } - }; + let handle = get_handle(handle_id)?; - match mode { - WriteMode::Passthrough(handle) => unsafe { + // At the time of writing, console IO on Windows is high latency: + // Windows has no TTY buffers, making every call a fully synchronous RPC. + // So, while always deferring to `write_console_utf16` wastes valuable heap for + // our UTF16 conversion, it handily beats doing 2 (or more) costly RPC round-trips. + if !is_console(handle) { + unsafe { let handle = Handle::from_raw_handle(handle); let ret = handle.write(data); let _ = handle.into_raw_handle(); // Don't close the handle ret - }, - WriteMode::Utf16Console(handle) => write_console_utf16(data, incomplete_utf8, handle), + } + } else { + write_console_utf16(data, incomplete_utf8, handle) } } @@ -164,141 +138,125 @@ fn write_console_utf16( incomplete_utf8: &mut IncompleteUtf8, handle: c::HANDLE, ) -> io::Result { - if incomplete_utf8.len > 0 { - assert!( - incomplete_utf8.len < 4, - "Unexpected number of bytes for incomplete UTF-8 codepoint." - ); - if data[0] >> 6 != 0b10 { - // not a continuation byte - reject - incomplete_utf8.len = 0; - return Err(io::const_error!( - io::ErrorKind::InvalidData, - "Windows stdio in console mode does not support writing non-UTF-8 byte sequences", - )); - } - incomplete_utf8.bytes[incomplete_utf8.len as usize] = data[0]; - incomplete_utf8.len += 1; - let char_width = utf8_char_width(incomplete_utf8.bytes[0]); - if (incomplete_utf8.len as usize) < char_width { - // more bytes needed - return Ok(1); - } - let s = str::from_utf8(&incomplete_utf8.bytes[0..incomplete_utf8.len as usize]); - incomplete_utf8.len = 0; - match s { - Ok(s) => { - assert_eq!(char_width, s.len()); - let written = write_valid_utf8_to_console(handle, s)?; - assert_eq!(written, s.len()); // guaranteed by write_valid_utf8_to_console() for single codepoint writes - return Ok(1); - } - Err(_) => { - return Err(io::const_error!( - io::ErrorKind::InvalidData, - "Windows stdio in console mode does not support writing non-UTF-8 byte sequences", - )); - } - } + // As stated for `MAX_BUFFER_SIZE`, Windows 7 does not support large writes at all. + // --> Fall back to the small-writes stack-alloc path. + #[cfg(not(target_vendor = "win7"))] + if data.len() > MAX_BUFFER_SIZE + && let capacity = data.len().min(MAX_WRITE_HEAP_LEN) + && let Ok(mut utf16) = Box::<[u16], _>::try_new_uninit_slice_in(capacity, System) + { + return write_console_with_buffer(data, incomplete_utf8, &mut utf16, handle); } - // As the console is meant for presenting text, we assume bytes of `data` are encoded as UTF-8, - // which needs to be encoded as UTF-16. - // - // If the data is not valid UTF-8 we write out as many bytes as are valid. - // If the first byte is invalid it is either first byte of a multi-byte sequence but the - // provided byte slice is too short or it is the first byte of an invalid multi-byte sequence. - let len = cmp::min(data.len(), MAX_BUFFER_SIZE / 2); - let utf8 = match str::from_utf8(&data[..len]) { - Ok(s) => s, - Err(ref e) if e.valid_up_to() == 0 => { - let first_byte_char_width = utf8_char_width(data[0]); - if first_byte_char_width > 1 && data.len() < first_byte_char_width { - incomplete_utf8.bytes[0] = data[0]; - incomplete_utf8.len = 1; - return Ok(1); - } else { - return Err(io::const_error!( - io::ErrorKind::InvalidData, - "Windows stdio in console mode does not support writing non-UTF-8 byte sequences", - )); - } - } - Err(e) => str::from_utf8(&data[..e.valid_up_to()]).unwrap(), - }; + write_console_stack(data, incomplete_utf8, handle) +} - write_valid_utf8_to_console(handle, utf8) +#[cfg_attr(not(target_vendor = "win7"), inline(never))] // Avoid stack allocations unless called +fn write_console_stack( + data: &[u8], + incomplete_utf8: &mut IncompleteUtf8, + handle: c::HANDLE, +) -> io::Result { + let mut utf16 = [const { MaybeUninit::uninit() }; MAX_BUFFER_SIZE]; + write_console_with_buffer(data, incomplete_utf8, &mut utf16, handle) } -fn write_valid_utf8_to_console(handle: c::HANDLE, utf8: &str) -> io::Result { - debug_assert!(!utf8.is_empty()); - - let mut utf16 = [MaybeUninit::::uninit(); MAX_BUFFER_SIZE / 2]; - let utf8 = &utf8[..utf8.floor_char_boundary(utf16.len())]; - - let utf16: &[u16] = unsafe { - // Note that this theoretically checks validity twice in the (most common) case - // where the underlying byte sequence is valid utf-8 (given the check in `write()`). - let result = c::MultiByteToWideChar( - c::CP_UTF8, // CodePage - c::MB_ERR_INVALID_CHARS, // dwFlags - utf8.as_ptr().cast::(), // lpMultiByteStr - utf8.len() as i32, // cbMultiByte - utf16.as_mut_ptr() as *mut c::WCHAR, // lpWideCharStr - utf16.len() as i32, // cchWideChar - ); - assert!(result != 0, "Unexpected error in MultiByteToWideChar"); - - // Safety: MultiByteToWideChar initializes `result` values. - utf16[..result as usize].assume_init_ref() - }; +fn write_console_with_buffer( + data: &[u8], + incomplete_utf8: &mut IncompleteUtf8, + utf16: &mut [MaybeUninit], + handle: c::HANDLE, +) -> io::Result { + let mut written = 0; + while written < data.len() { + match write_console_chunk(&data[written..], incomplete_utf8, utf16, handle) { + Ok(0) => break, + Ok(consumed) => written += consumed, + Err(error) if written == 0 => return Err(error), + Err(_) => break, + } + } + Ok(written) +} - let mut written = write_u16s(handle, utf16)?; +fn write_console_chunk( + data: &[u8], + incomplete_utf8: &mut IncompleteUtf8, + utf16: &mut [MaybeUninit], + handle: c::HANDLE, +) -> io::Result { + debug_assert!(utf16.len() >= 2, "not enough space for a full codepoint"); + debug_assert!(utf16.len() >= 1024, "unreasonably small buffer"); - // Figure out how many bytes of as UTF-8 were written away as UTF-16. - if written == utf16.len() { - Ok(utf8.len()) - } else { - // Make sure we didn't end up writing only half of a surrogate pair (even though the chance - // is tiny). Because it is not possible for user code to re-slice `data` in such a way that - // a missing surrogate can be produced (and also because of the UTF-8 validation above), - // write the missing surrogate out now. - // Buffering it would mean we have to lie about the number of bytes written. - let first_code_unit_remaining = utf16[written]; - if matches!(first_code_unit_remaining, 0xDC00..=0xDFFF) { - // low surrogate - // We just hope this works, and give up otherwise - let _ = write_u16s(handle, &utf16[written..written + 1]); - written += 1; + let mut chunk_len = data.len().min(utf16.len() - incomplete_utf8.len as usize); + if chunk_len < data.len() { + chunk_len = incomplete_utf8_suffix_start(&data[..chunk_len]); + } + let data = &data[..chunk_len]; + let mut utf16_len = 0; + let mut remaining = data; + + // Join the incomplete sequence with the new data. + if incomplete_utf8.len > 0 { + let mut len = incomplete_utf8.len as usize; + let width = utf8_char_width(incomplete_utf8.bytes[0]); + + // Accumulate until we have a complete sequence, + // or something recognizable as a invalid sequence. + while len < width + && let Some((&b, rest)) = remaining.split_first() + && b & 0xc0 == 0x80 + { + incomplete_utf8.bytes[len] = b; + len += 1; + remaining = rest; } - // Calculate the number of bytes of `utf8` that were actually written. - let mut count = 0; - for ch in utf16[..written].iter() { - count += match ch { - 0x0000..=0x007F => 1, - 0x0080..=0x07FF => 2, - 0xDC00..=0xDFFF => 1, // Low surrogate. We already counted 3 bytes for the other. - _ => 3, - }; + + // If we consumed all of `data` without completing the sequence we're done. + if len < width && remaining.is_empty() { + incomplete_utf8.len = len as u8; + return Ok(data.len()); } - debug_assert!(String::from_utf16(&utf16[..written]).unwrap() == utf8[..count]); - Ok(count) + + utf16_len = utf8_to_utf16(&incomplete_utf8.bytes[..len], utf16)?; + } + + // Strip off any incomplete, trailing UTF8 sequence. + let (remaining, trailing) = remaining.split_at(incomplete_utf8_suffix_start(remaining)); + + // Convert the bulk of the data to UTF16. + if !remaining.is_empty() { + utf16_len += utf8_to_utf16(remaining, &mut utf16[utf16_len..])?; + } + + if utf16_len != 0 { + // NOTE: WriteConsoleW does not result in short writes. + let mut written = 0; + cvt(unsafe { + c::WriteConsoleW( + handle, + utf16.as_ptr().cast(), + utf16_len as u32, + &mut written, + ptr::null_mut(), + ) + })?; } + + incomplete_utf8.bytes[..trailing.len()].copy_from_slice(trailing); + incomplete_utf8.len = trailing.len() as u8; + Ok(data.len()) } -fn write_u16s(handle: c::HANDLE, data: &[u16]) -> io::Result { - debug_assert!(data.len() < u32::MAX as usize); - let mut written = 0; - cvt(unsafe { - c::WriteConsoleW( - handle, - data.as_ptr().cast::(), - data.len() as u32, - &mut written, - ptr::null_mut(), - ) - })?; - Ok(written as usize) +fn incomplete_utf8_suffix_start(bytes: &[u8]) -> usize { + for length in 1..=3.min(bytes.len()) { + let start = bytes.len() - length; + let byte = bytes[start]; + if byte & 0xc0 != 0x80 { + return if utf8_char_width(byte) > length { start } else { bytes.len() }; + } + } + bytes.len() } impl Stdin { @@ -451,28 +409,62 @@ fn utf16_to_utf8(utf16: &[u16], utf8: &mut [u8]) -> io::Result { let result = unsafe { c::WideCharToMultiByte( - c::CP_UTF8, // CodePage - c::WC_ERR_INVALID_CHARS, // dwFlags - utf16.as_ptr(), // lpWideCharStr - utf16.len() as i32, // cchWideChar - utf8.as_mut_ptr(), // lpMultiByteStr - utf8.len() as i32, // cbMultiByte - ptr::null(), // lpDefaultChar - ptr::null_mut(), // lpUsedDefaultChar + c::CP_UTF8, // CodePage + 0, // dwFlags + utf16.as_ptr(), // lpWideCharStr + utf16.len() as i32, // cchWideChar + utf8.as_mut_ptr(), // lpMultiByteStr + utf8.len() as i32, // cbMultiByte + ptr::null(), // lpDefaultChar + ptr::null_mut(), // lpUsedDefaultChar ) }; if result == 0 { - // We can't really do any better than forget all data and return an error. - Err(io::const_error!( - io::ErrorKind::InvalidData, - "Windows stdin in console mode does not support non-UTF-16 input; \ - encountered unpaired surrogate", - )) + return Err(io::Error::last_os_error()); } else { Ok(result as usize) } } +fn utf8_to_utf16(data: &[u8], utf16: &mut [MaybeUninit]) -> io::Result { + debug_assert!(!data.is_empty()); + + if utf16.is_empty() { + return Ok(0); + } + + // Fun fact: As is often the case, Windows' algorithm predates Unicode's recommendation. + // The current behavior was implemented for Windows Vista (2007) and Unicode's recommendation + // for "Best Practices for Using U+FFFD" came with Unicode 5.2 (2009) [^1]. So, now if a + // continuation byte violates a boundary limit, it should restore it [^2], but this consumes it. + // For instance, `ED A0 80` produces two U+FFFD with this API, while it should be three. + // In practice this difference bears no meaning for console output. + // + // NOTE: However, it means we have to be careful about the treatment of `incomplete_utf8`, + // because the behavior of `MultiByteToWideChar` could theoretically change in the future. + // + // `MultiByteToWideChar` is technically a fancy wrapper around `RtlUTF8ToUnicodeN`, + // whose MSDN is a bit more enlightening about the behavior of the conversion. + // Despite its name, it's not a syscall, but a simple function inside ntdll.dll. + // + // [^1]: https://www.unicode.org/review/pr-121.html + // [^2]: https://encoding.spec.whatwg.org/#utf-8-decoder + let len = unsafe { + c::MultiByteToWideChar( + c::CP_UTF8, + 0, + data.as_ptr().cast(), + data.len() as i32, + utf16.as_mut_ptr().cast(), + utf16.len() as i32, + ) + }; + if len <= 0 { + return Err(io::Error::last_os_error()); + } + Ok(len as usize) +} + impl IncompleteUtf8 { pub const fn new() -> IncompleteUtf8 { IncompleteUtf8 { bytes: [0; char::MAX_LEN_UTF8], len: 0 } @@ -481,21 +473,13 @@ impl IncompleteUtf8 { impl Stdout { pub const fn new() -> Stdout { - Stdout { incomplete_utf8: IncompleteUtf8::new(), write_mode: None } - } - - /// Forgets the cached stream state, so the next write re-queries the OS. - /// Called when a new lock session begins, so that changing the process - /// stdio handles (e.g. `SetStdHandle`) between lock sessions keeps - /// working (see #40490). - pub fn refresh(&mut self) { - self.write_mode = None; + Stdout { incomplete_utf8: IncompleteUtf8::new() } } } impl io::Write for Stdout { fn write(&mut self, buf: &[u8]) -> io::Result { - write(c::STD_OUTPUT_HANDLE, buf, &mut self.incomplete_utf8, &mut self.write_mode) + write(c::STD_OUTPUT_HANDLE, buf, &mut self.incomplete_utf8) } fn flush(&mut self) -> io::Result<()> { @@ -505,18 +489,13 @@ impl io::Write for Stdout { impl Stderr { pub const fn new() -> Stderr { - Stderr { incomplete_utf8: IncompleteUtf8::new(), write_mode: None } - } - - /// See `Stdout::refresh`. - pub fn refresh(&mut self) { - self.write_mode = None; + Stderr { incomplete_utf8: IncompleteUtf8::new() } } } impl io::Write for Stderr { fn write(&mut self, buf: &[u8]) -> io::Result { - write(c::STD_ERROR_HANDLE, buf, &mut self.incomplete_utf8, &mut self.write_mode) + write(c::STD_ERROR_HANDLE, buf, &mut self.incomplete_utf8) } fn flush(&mut self) -> io::Result<()> { diff --git a/library/std/src/sys/sync/condvar/no_threads.rs b/library/std/src/sys/sync/condvar/no_threads.rs index 9dbba490b2851..98ca2288f74a2 100644 --- a/library/std/src/sys/sync/condvar/no_threads.rs +++ b/library/std/src/sys/sync/condvar/no_threads.rs @@ -2,7 +2,13 @@ use crate::sys::sync::Mutex; use crate::thread::sleep; use crate::time::Duration; -#[cfg(target_has_threads)] +#[cfg(all( + target_has_threads, + // VEXos is only "target_has_threads" because it allows user interrupt handlers which preempt + // the main thread, but they are forbidden from accessing sync primitives (the same as UNIX + // signal handlers). + not(target_os = "vexos"), +))] compile_error!("Using no_threads implementation on a target with threads"); pub struct Condvar {} diff --git a/library/std/src/sys/sync/mutex/no_threads.rs b/library/std/src/sys/sync/mutex/no_threads.rs index a87c2747451c5..d031c0e7e1dcf 100644 --- a/library/std/src/sys/sync/mutex/no_threads.rs +++ b/library/std/src/sys/sync/mutex/no_threads.rs @@ -1,6 +1,12 @@ use crate::cell::Cell; -#[cfg(target_has_threads)] +#[cfg(all( + target_has_threads, + // VEXos is only "target_has_threads" because it allows user interrupt handlers which preempt + // the main thread, but they are forbidden from accessing sync primitives (the same as UNIX + // signal handlers). + not(target_os = "vexos"), +))] compile_error!("Using no_threads implementation on a target with threads"); pub struct Mutex { diff --git a/library/std/src/sys/sync/once/no_threads.rs b/library/std/src/sys/sync/once/no_threads.rs index 31c9aedb98f96..725ec28f7cd4f 100644 --- a/library/std/src/sys/sync/once/no_threads.rs +++ b/library/std/src/sys/sync/once/no_threads.rs @@ -2,7 +2,13 @@ use crate::cell::Cell; use crate::sync as public; use crate::sync::once::OnceExclusiveState; -#[cfg(target_has_threads)] +#[cfg(all( + target_has_threads, + // VEXos is only "target_has_threads" because it allows user interrupt handlers which preempt + // the main thread, but they are forbidden from accessing sync primitives (the same as UNIX + // signal handlers). + not(target_os = "vexos"), +))] compile_error!("Using no_threads implementation on a target with threads"); pub struct Once { diff --git a/library/std/src/sys/sync/rwlock/no_threads.rs b/library/std/src/sys/sync/rwlock/no_threads.rs index efa2629b3110d..e7e845cc785c7 100644 --- a/library/std/src/sys/sync/rwlock/no_threads.rs +++ b/library/std/src/sys/sync/rwlock/no_threads.rs @@ -1,6 +1,12 @@ use crate::cell::Cell; -#[cfg(target_has_threads)] +#[cfg(all( + target_has_threads, + // VEXos is only "target_has_threads" because it allows user interrupt handlers which preempt + // the main thread, but they are forbidden from accessing sync primitives (the same as UNIX + // signal handlers). + not(target_os = "vexos"), +))] compile_error!("Using no_threads implementation on a target with threads"); pub struct RwLock { diff --git a/library/std/src/sys/sync/thread_parking/mod.rs b/library/std/src/sys/sync/thread_parking/mod.rs index f1385ef7bdde6..dffa0ade947ee 100644 --- a/library/std/src/sys/sync/thread_parking/mod.rs +++ b/library/std/src/sys/sync/thread_parking/mod.rs @@ -27,8 +27,7 @@ cfg_select! { mod windows7; pub use windows7::Parker; } - all(target_vendor = "apple", not(miri)) => { - // Doesn't work in Miri, see . + target_vendor = "apple" => { mod darwin; pub use darwin::Parker; } diff --git a/library/std/src/sys/thread_local/destructors/linux_like.rs b/library/std/src/sys/thread_local/destructors/linux_like.rs index d7cbaeb89f42e..5f2e454b99e9e 100644 --- a/library/std/src/sys/thread_local/destructors/linux_like.rs +++ b/library/std/src/sys/thread_local/destructors/linux_like.rs @@ -37,14 +37,17 @@ pub unsafe fn register(t: *mut u8, dtor: unsafe extern "C" fn(*mut u8)) { >; } - if let Some(f) = unsafe { __cxa_thread_atexit_impl } { + if let Some(f) = unsafe { __cxa_thread_atexit_impl } + && let dso = unsafe { __dso_handle } + && !dso.is_null() + { unsafe { f( transmute::( dtor, ), t.cast(), - (&raw const __dso_handle) as *mut _, + dso as *mut _, ); } } else { diff --git a/library/std/src/sys/thread_local/no_threads.rs b/library/std/src/sys/thread_local/no_threads.rs index a3d50d05eeec8..c8d40a5f4e5d7 100644 --- a/library/std/src/sys/thread_local/no_threads.rs +++ b/library/std/src/sys/thread_local/no_threads.rs @@ -5,7 +5,14 @@ use crate::cell::{Cell, UnsafeCell}; use crate::mem::MaybeUninit; use crate::ptr; -#[cfg(target_has_threads)] +#[cfg(all( + target_has_threads, + // VEXos is only "target_has_threads" because it allows user interrupt handlers which preempt + // the main thread, but they are forbidden from accessing TLS (like UNIX signal handlers). + // Since there is only one context that's allowed to access thread locals, it's sound to use the + // no_threads implementation. + not(target_os = "vexos"), +))] compile_error!("Using no_threads implementation on a target with threads"); #[doc(hidden)] diff --git a/library/std/tests/run-time-detect.rs b/library/std/tests/run-time-detect.rs index 5506fd469732d..d202a2e1bb800 100644 --- a/library/std/tests/run-time-detect.rs +++ b/library/std/tests/run-time-detect.rs @@ -9,7 +9,10 @@ feature(stdarch_aarch64_feature_detection) )] #![cfg_attr( - all(target_arch = "powerpc", target_os = "linux"), + all( + any(target_arch = "powerpc", target_arch = "powerpc64"), + any(target_os = "linux", target_os = "freebsd", target_os = "openbsd"), + ), feature(stdarch_powerpc_feature_detection) )] #![cfg_attr( @@ -111,7 +114,10 @@ fn aarch64_linux() { } #[test] -#[cfg(all(target_arch = "powerpc", target_os = "linux"))] +#[cfg(all( + any(target_arch = "powerpc", target_arch = "powerpc64"), + any(target_os = "linux", target_os = "freebsd", target_os = "openbsd"), +))] fn powerpc_linux() { use std::arch::is_powerpc_feature_detected; // tidy-alphabetical-start @@ -121,17 +127,6 @@ fn powerpc_linux() { // tidy-alphabetical-end } -#[test] -#[cfg(all(target_arch = "powerpc64", target_os = "linux"))] -fn powerpc64_linux() { - use std::arch::is_powerpc64_feature_detected; - // tidy-alphabetical-start - println!("altivec: {}", is_powerpc64_feature_detected!("altivec")); - println!("power8: {}", is_powerpc64_feature_detected!("power8")); - println!("vsx: {}", is_powerpc64_feature_detected!("vsx")); - // tidy-alphabetical-end -} - #[test] #[cfg(all(target_arch = "s390x", target_os = "linux"))] fn s390x_linux() { diff --git a/library/std/tests/sync/oneshot.rs b/library/std/tests/sync/oneshot.rs index 0197369869d80..b7bdaedc62d36 100644 --- a/library/std/tests/sync/oneshot.rs +++ b/library/std/tests/sync/oneshot.rs @@ -9,6 +9,7 @@ use std::{mem, thread}; #[test] fn is_ready_after_send() { let (sender, receiver) = oneshot::channel(); + assert!(!sender.is_closed()); assert_eq!(receiver.is_ready(), Ok(false)); sender.send(String::from("hello")).unwrap(); @@ -126,6 +127,7 @@ fn send_error() { mem::drop(receiver); + assert!(sender.is_closed()); let send_error = sender.send(32u128).unwrap_err(); assert_eq!(send_error.0, 32); } diff --git a/library/std_detect/src/detect/arch/mod.rs b/library/std_detect/src/detect/arch/mod.rs index 2ca4cbd1e3fa0..d3ef2b65c87aa 100644 --- a/library/std_detect/src/detect/arch/mod.rs +++ b/library/std_detect/src/detect/arch/mod.rs @@ -12,8 +12,6 @@ mod riscv; #[macro_use] mod powerpc; #[macro_use] -mod powerpc64; -#[macro_use] mod mips; #[macro_use] mod mips64; @@ -39,14 +37,10 @@ cfg_select! { #[unstable(feature = "stdarch_riscv_feature_detection", issue = "111192")] pub use riscv::*; } - target_arch = "powerpc" => { + any(target_arch = "powerpc", target_arch = "powerpc64") => { #[unstable(feature = "stdarch_powerpc_feature_detection", issue = "111191")] pub use powerpc::*; } - target_arch = "powerpc64" => { - #[unstable(feature = "stdarch_powerpc_feature_detection", issue = "111191")] - pub use powerpc64::*; - } target_arch = "mips" => { #[unstable(feature = "stdarch_mips_feature_detection", issue = "111188")] pub use mips::*; diff --git a/library/std_detect/src/detect/arch/powerpc.rs b/library/std_detect/src/detect/arch/powerpc.rs index be2db0b81c2fc..02167e5935bdf 100644 --- a/library/std_detect/src/detect/arch/powerpc.rs +++ b/library/std_detect/src/detect/arch/powerpc.rs @@ -2,7 +2,7 @@ features! { @TARGET: powerpc; - @CFG: target_arch = "powerpc"; + @CFG: any(target_arch = "powerpc", target_arch = "powerpc64"); @MACRO_NAME: is_powerpc_feature_detected; @MACRO_ATTRS: /// Check for the presence of a CPU feature at runtime. diff --git a/library/std_detect/src/detect/arch/powerpc64.rs b/library/std_detect/src/detect/arch/powerpc64.rs deleted file mode 100644 index 98e8d5f32b75c..0000000000000 --- a/library/std_detect/src/detect/arch/powerpc64.rs +++ /dev/null @@ -1,33 +0,0 @@ -//! Run-time feature detection on PowerPC64. - -features! { - @TARGET: powerpc64; - @CFG: target_arch = "powerpc64"; - @MACRO_NAME: is_powerpc64_feature_detected; - @MACRO_ATTRS: - /// Check for the presence of a CPU feature at runtime. - /// - /// When the feature is known to be enabled at compile time (e.g. via `-Ctarget-feature`) - /// the macro expands to `true`. - #[unstable(feature = "stdarch_powerpc_feature_detection", issue = "111191")] - @FEATURE: #[unstable(feature = "stdarch_powerpc_feature_detection", issue = "111191")] altivec: "altivec"; - /// Altivec - @FEATURE: #[unstable(feature = "stdarch_powerpc_feature_detection", issue = "111191")] vsx: "vsx"; - /// VSX - @FEATURE: #[unstable(feature = "stdarch_powerpc_feature_detection", issue = "111191")] power8: "power8"; - without cfg check: true; - /// Power8 - @FEATURE: #[unstable(feature = "stdarch_powerpc_feature_detection", issue = "111191")] power8_altivec: "power8-altivec"; - /// Power8 altivec - @FEATURE: #[unstable(feature = "stdarch_powerpc_feature_detection", issue = "111191")] power8_vector: "power8-vector"; - /// Power8 vector - @FEATURE: #[unstable(feature = "stdarch_powerpc_feature_detection", issue = "111191")] power8_crypto: "power8-crypto"; - /// Power8 crypto - @FEATURE: #[unstable(feature = "stdarch_powerpc_feature_detection", issue = "111191")] power9: "power9"; - without cfg check: true; - /// Power9 - @FEATURE: #[unstable(feature = "stdarch_powerpc_feature_detection", issue = "111191")] power9_altivec: "power9-altivec"; - /// Power9 altivec - @FEATURE: #[unstable(feature = "stdarch_powerpc_feature_detection", issue = "111191")] power9_vector: "power9-vector"; - /// Power9 vector -} diff --git a/library/std_detect/src/detect/os/freebsd/powerpc.rs b/library/std_detect/src/detect/os/freebsd/powerpc.rs index d03af68cd0815..e41c37f1a5f73 100644 --- a/library/std_detect/src/detect/os/freebsd/powerpc.rs +++ b/library/std_detect/src/detect/os/freebsd/powerpc.rs @@ -1,8 +1,16 @@ //! Run-time feature detection for PowerPC on FreeBSD. +//! +//! Constants for both 32- and 64-bit come from +//! . use super::auxvec; use crate::detect::{Feature, cache}; +const PPC_FEATURE_HAS_ALTIVEC: usize = 0x10000000; +const PPC_FEATURE_HAS_VSX: usize = 0x00000080; +/// ISA 2.07, typically power8 +const PPC_FEATURE2_ARCH_2_07: usize = 0x80000000; + pub(crate) fn detect_features() -> cache::Initializer { let mut value = cache::Initializer::default(); let enable_feature = |value: &mut cache::Initializer, f, enable| { @@ -12,9 +20,9 @@ pub(crate) fn detect_features() -> cache::Initializer { }; if let Ok(auxv) = auxvec::auxv() { - enable_feature(&mut value, Feature::altivec, auxv.hwcap & 0x10000000 != 0); - enable_feature(&mut value, Feature::vsx, auxv.hwcap & 0x00000080 != 0); - enable_feature(&mut value, Feature::power8, auxv.hwcap2 & 0x80000000 != 0); + enable_feature(&mut value, Feature::altivec, auxv.hwcap & PPC_FEATURE_HAS_ALTIVEC != 0); + enable_feature(&mut value, Feature::vsx, auxv.hwcap & PPC_FEATURE_HAS_VSX != 0); + enable_feature(&mut value, Feature::power8, auxv.hwcap2 & PPC_FEATURE2_ARCH_2_07 != 0); return value; } value diff --git a/library/std_detect/src/detect/os/linux/powerpc.rs b/library/std_detect/src/detect/os/linux/powerpc.rs index 6a4f7e715d932..73d5f3d1f7347 100644 --- a/library/std_detect/src/detect/os/linux/powerpc.rs +++ b/library/std_detect/src/detect/os/linux/powerpc.rs @@ -1,8 +1,18 @@ //! Run-time feature detection for PowerPC on Linux. +//! +//! Constants for both 32- and 64-bit come from +//! . use super::auxvec; use crate::detect::{Feature, cache}; +const PPC_FEATURE_HAS_ALTIVEC: usize = 0x10000000; +const PPC_FEATURE_HAS_VSX: usize = 0x00000080; +/// ISA 2.07, typically power8 +const PPC_FEATURE2_ARCH_2_07: usize = 0x80000000; +/// ISA 3.00, typically power9 +const PPC_FEATURE2_ARCH_3_00: usize = 0x00800000; + /// Try to read the features from the auxiliary vector. pub(crate) fn detect_features() -> cache::Initializer { let mut value = cache::Initializer::default(); @@ -12,20 +22,20 @@ pub(crate) fn detect_features() -> cache::Initializer { } }; - // The values are part of the platform-specific [asm/cputable.h][cputable] - // - // [cputable]: https://github.com/torvalds/linux/blob/master/arch/powerpc/include/uapi/asm/cputable.h if let Ok(auxv) = auxvec::auxv() { // note: the PowerPC values are the mask to do the test (instead of the // index of the bit to test like in ARM and Aarch64) - enable_feature(&mut value, Feature::altivec, auxv.hwcap & 0x10000000 != 0); - enable_feature(&mut value, Feature::vsx, auxv.hwcap & 0x00000080 != 0); - let power8_features = auxv.hwcap2 & 0x80000000 != 0; + enable_feature(&mut value, Feature::altivec, auxv.hwcap & PPC_FEATURE_HAS_ALTIVEC != 0); + enable_feature(&mut value, Feature::vsx, auxv.hwcap & PPC_FEATURE_HAS_VSX != 0); + // FIXME(powerpc_target_feature): this code assumes the ISA hwcaps imply vsx/altivec, but + // this is not a correct assumption. See rust-lang/compiler-team#1046 for + // a possible resolution. + let power8_features = auxv.hwcap2 & PPC_FEATURE2_ARCH_2_07 != 0; enable_feature(&mut value, Feature::power8, power8_features); enable_feature(&mut value, Feature::power8_altivec, power8_features); enable_feature(&mut value, Feature::power8_crypto, power8_features); enable_feature(&mut value, Feature::power8_vector, power8_features); - let power9_features = auxv.hwcap2 & 0x00800000 != 0; + let power9_features = auxv.hwcap2 & PPC_FEATURE2_ARCH_3_00 != 0; enable_feature(&mut value, Feature::power9, power9_features); enable_feature(&mut value, Feature::power9_altivec, power9_features); enable_feature(&mut value, Feature::power9_vector, power9_features); diff --git a/library/std_detect/src/detect/os/openbsd/powerpc.rs b/library/std_detect/src/detect/os/openbsd/powerpc.rs index dd98ab2a3f76e..7a684cd273c64 100644 --- a/library/std_detect/src/detect/os/openbsd/powerpc.rs +++ b/library/std_detect/src/detect/os/openbsd/powerpc.rs @@ -1,8 +1,19 @@ //! Run-time feature detection for PowerPC on OpenBSD. +//! +//! Constants for 32-bit are located at +//! , and for 64-bit +//! . use super::auxvec; use crate::detect::{Feature, cache}; +/* Note that most constants are the same for 32- and 64-bit, but not all are */ + +const PPC_FEATURE_HAS_ALTIVEC: usize = 0x10000000; +const PPC_FEATURE_HAS_VSX: usize = 0x00000080; +/// ISA 2.07, typically power8 +const PPC_FEATURE2_ARCH_2_07: usize = 0x80000000; + pub(crate) fn detect_features() -> cache::Initializer { let mut value = cache::Initializer::default(); let enable_feature = |value: &mut cache::Initializer, f, enable| { @@ -12,9 +23,9 @@ pub(crate) fn detect_features() -> cache::Initializer { }; if let Ok(auxv) = auxvec::auxv() { - enable_feature(&mut value, Feature::altivec, auxv.hwcap & 0x10000000 != 0); - enable_feature(&mut value, Feature::vsx, auxv.hwcap & 0x00000080 != 0); - enable_feature(&mut value, Feature::power8, auxv.hwcap2 & 0x80000000 != 0); + enable_feature(&mut value, Feature::altivec, auxv.hwcap & PPC_FEATURE_HAS_ALTIVEC != 0); + enable_feature(&mut value, Feature::vsx, auxv.hwcap & PPC_FEATURE_HAS_VSX != 0); + enable_feature(&mut value, Feature::power8, auxv.hwcap2 & PPC_FEATURE2_ARCH_2_07 != 0); return value; } value diff --git a/library/std_detect/src/lib.rs b/library/std_detect/src/lib.rs index f9d79df670a0f..d5805875ad9d6 100644 --- a/library/std_detect/src/lib.rs +++ b/library/std_detect/src/lib.rs @@ -10,7 +10,6 @@ //! * `mips`: [`is_mips_feature_detected`] //! * `mips64`: [`is_mips64_feature_detected`] //! * `powerpc`: [`is_powerpc_feature_detected`] -//! * `powerpc64`: [`is_powerpc64_feature_detected`] //! * `loongarch`: [`is_loongarch_feature_detected`] //! * `s390x`: [`is_s390x_feature_detected`] diff --git a/library/std_detect/tests/cpu-detection.rs b/library/std_detect/tests/cpu-detection.rs index a381d84cfbbc3..9e714679284b0 100644 --- a/library/std_detect/tests/cpu-detection.rs +++ b/library/std_detect/tests/cpu-detection.rs @@ -317,24 +317,16 @@ fn riscv_linux() { println!("p: {}", is_riscv_feature_detected!("p")); } -#[test] -#[cfg(all(target_arch = "powerpc", target_os = "linux"))] -fn powerpc_linux() { - println!("altivec: {}", is_powerpc_feature_detected!("altivec")); - println!("vsx: {}", is_powerpc_feature_detected!("vsx")); - println!("power8: {}", is_powerpc_feature_detected!("power8")); -} - #[test] #[cfg(all( - target_arch = "powerpc64", + any(target_arch = "powerpc", target_arch = "powerpc64"), any(target_os = "linux", target_os = "freebsd", target_os = "openbsd"), ))] -fn powerpc64_linux_or_bsd() { - println!("altivec: {}", is_powerpc64_feature_detected!("altivec")); - println!("vsx: {}", is_powerpc64_feature_detected!("vsx")); - println!("power8: {}", is_powerpc64_feature_detected!("power8")); - println!("power9: {}", is_powerpc64_feature_detected!("power9")); +fn powerpc_linux_or_bsd() { + println!("altivec: {}", is_powerpc_feature_detected!("altivec")); + println!("vsx: {}", is_powerpc_feature_detected!("vsx")); + println!("power8: {}", is_powerpc_feature_detected!("power8")); + println!("power9: {}", is_powerpc_feature_detected!("power9")); } #[test] diff --git a/library/std_detect/tests/macro_trailing_commas.rs b/library/std_detect/tests/macro_trailing_commas.rs index a60b34acb872f..f1d2973b961e8 100644 --- a/library/std_detect/tests/macro_trailing_commas.rs +++ b/library/std_detect/tests/macro_trailing_commas.rs @@ -75,19 +75,12 @@ fn loongarch() { } #[test] -#[cfg(target_arch = "powerpc")] +#[cfg(any(target_arch = "powerpc", target_arch = "powerpc64"))] fn powerpc() { let _ = is_powerpc_feature_detected!("altivec"); let _ = is_powerpc_feature_detected!("altivec",); } -#[test] -#[cfg(target_arch = "powerpc64")] -fn powerpc64() { - let _ = is_powerpc64_feature_detected!("altivec"); - let _ = is_powerpc64_feature_detected!("altivec",); -} - #[test] #[cfg(any(target_arch = "riscv32", target_arch = "riscv64"))] fn riscv() { diff --git a/library/stdarch/.github/workflows/main.yml b/library/stdarch/.github/workflows/main.yml index d7539fe2e8ebd..bf8ad2e293e7f 100644 --- a/library/stdarch/.github/workflows/main.yml +++ b/library/stdarch/.github/workflows/main.yml @@ -8,7 +8,7 @@ jobs: name: Check Style runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Install Rust run: rustup update nightly --no-self-update && rustup default nightly - run: ci/style.sh @@ -18,7 +18,7 @@ jobs: needs: [ style ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Install Rust run: rustup update nightly --no-self-update && rustup default nightly - run: ci/dox.sh @@ -30,7 +30,7 @@ jobs: needs: [ style ] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Install Rust run: rustup update nightly --no-self-update && rustup default nightly - run: cargo test --manifest-path crates/stdarch-verify/Cargo.toml @@ -218,7 +218,7 @@ jobs: # build_std: true steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Install Rust run: | rustup update nightly --no-self-update @@ -298,7 +298,7 @@ jobs: - target: armv7-unknown-linux-gnueabihf cc: gcc steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Install Rust run: | rustup update nightly --no-self-update @@ -326,7 +326,7 @@ jobs: env: STDARCH_GEN_MODE: check steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Install Rust run: rustup update nightly && rustup default nightly && rustup component add rustfmt - name: Check arm spec @@ -352,7 +352,7 @@ jobs: name: Run some tests with miri runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - name: Install Rust run: rustup update nightly && rustup default nightly && rustup component add miri - name: Run miri tests diff --git a/library/stdarch/.github/workflows/rustc-pull.yml b/library/stdarch/.github/workflows/rustc-pull.yml index d2feb1add6344..96cf9285ec10b 100644 --- a/library/stdarch/.github/workflows/rustc-pull.yml +++ b/library/stdarch/.github/workflows/rustc-pull.yml @@ -10,6 +10,7 @@ on: jobs: pull: if: github.repository == 'rust-lang/stdarch' + # zizmor: ignore[unpinned-uses] We explicitly want to track the main branch. uses: rust-lang/josh-sync/.github/workflows/rustc-pull.yml@main with: github-app-id: ${{ vars.APP_CLIENT_ID }} diff --git a/library/stdarch/crates/core_arch/src/loongarch64/lasx/generated.rs b/library/stdarch/crates/core_arch/src/loongarch64/lasx/generated.rs index cfc4aced34173..70560f8b5a68d 100644 --- a/library/stdarch/crates/core_arch/src/loongarch64/lasx/generated.rs +++ b/library/stdarch/crates/core_arch/src/loongarch64/lasx/generated.rs @@ -91,6 +91,22 @@ unsafe extern "llvm-intrinsic" { fn __lasx_xvsat_wu(a: __v8u32, b: u32) -> __v8u32; #[link_name = "llvm.loongarch.lasx.xvsat.du"] fn __lasx_xvsat_du(a: __v4u64, b: u32) -> __v4u64; + #[link_name = "llvm.loongarch.lasx.xvdiv.b"] + fn __lasx_xvdiv_b(a: __v32i8, b: __v32i8) -> __v32i8; + #[link_name = "llvm.loongarch.lasx.xvdiv.h"] + fn __lasx_xvdiv_h(a: __v16i16, b: __v16i16) -> __v16i16; + #[link_name = "llvm.loongarch.lasx.xvdiv.w"] + fn __lasx_xvdiv_w(a: __v8i32, b: __v8i32) -> __v8i32; + #[link_name = "llvm.loongarch.lasx.xvdiv.d"] + fn __lasx_xvdiv_d(a: __v4i64, b: __v4i64) -> __v4i64; + #[link_name = "llvm.loongarch.lasx.xvdiv.bu"] + fn __lasx_xvdiv_bu(a: __v32u8, b: __v32u8) -> __v32u8; + #[link_name = "llvm.loongarch.lasx.xvdiv.hu"] + fn __lasx_xvdiv_hu(a: __v16u16, b: __v16u16) -> __v16u16; + #[link_name = "llvm.loongarch.lasx.xvdiv.wu"] + fn __lasx_xvdiv_wu(a: __v8u32, b: __v8u32) -> __v8u32; + #[link_name = "llvm.loongarch.lasx.xvdiv.du"] + fn __lasx_xvdiv_du(a: __v4u64, b: __v4u64) -> __v4u64; #[link_name = "llvm.loongarch.lasx.xvhaddw.h.b"] fn __lasx_xvhaddw_h_b(a: __v32i8, b: __v32i8) -> __v16i16; #[link_name = "llvm.loongarch.lasx.xvhaddw.w.h"] @@ -115,6 +131,22 @@ unsafe extern "llvm-intrinsic" { fn __lasx_xvhsubw_wu_hu(a: __v16u16, b: __v16u16) -> __v8i32; #[link_name = "llvm.loongarch.lasx.xvhsubw.du.wu"] fn __lasx_xvhsubw_du_wu(a: __v8u32, b: __v8u32) -> __v4i64; + #[link_name = "llvm.loongarch.lasx.xvmod.b"] + fn __lasx_xvmod_b(a: __v32i8, b: __v32i8) -> __v32i8; + #[link_name = "llvm.loongarch.lasx.xvmod.h"] + fn __lasx_xvmod_h(a: __v16i16, b: __v16i16) -> __v16i16; + #[link_name = "llvm.loongarch.lasx.xvmod.w"] + fn __lasx_xvmod_w(a: __v8i32, b: __v8i32) -> __v8i32; + #[link_name = "llvm.loongarch.lasx.xvmod.d"] + fn __lasx_xvmod_d(a: __v4i64, b: __v4i64) -> __v4i64; + #[link_name = "llvm.loongarch.lasx.xvmod.bu"] + fn __lasx_xvmod_bu(a: __v32u8, b: __v32u8) -> __v32u8; + #[link_name = "llvm.loongarch.lasx.xvmod.hu"] + fn __lasx_xvmod_hu(a: __v16u16, b: __v16u16) -> __v16u16; + #[link_name = "llvm.loongarch.lasx.xvmod.wu"] + fn __lasx_xvmod_wu(a: __v8u32, b: __v8u32) -> __v8u32; + #[link_name = "llvm.loongarch.lasx.xvmod.du"] + fn __lasx_xvmod_du(a: __v4u64, b: __v4u64) -> __v4u64; #[link_name = "llvm.loongarch.lasx.xvshuf.b"] fn __lasx_xvshuf_b(a: __v32i8, b: __v32i8, c: __v32i8) -> __v32i8; #[link_name = "llvm.loongarch.lasx.xvshuf.h"] @@ -1251,6 +1283,62 @@ pub fn lasx_xvsat_du(a: m256i) -> m256i { unsafe { transmute(__lasx_xvsat_du(transmute(a), IMM6)) } } +#[inline] +#[target_feature(enable = "lasx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lasx_xvdiv_b(a: m256i, b: m256i) -> m256i { + unsafe { transmute(__lasx_xvdiv_b(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lasx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lasx_xvdiv_h(a: m256i, b: m256i) -> m256i { + unsafe { transmute(__lasx_xvdiv_h(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lasx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lasx_xvdiv_w(a: m256i, b: m256i) -> m256i { + unsafe { transmute(__lasx_xvdiv_w(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lasx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lasx_xvdiv_d(a: m256i, b: m256i) -> m256i { + unsafe { transmute(__lasx_xvdiv_d(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lasx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lasx_xvdiv_bu(a: m256i, b: m256i) -> m256i { + unsafe { transmute(__lasx_xvdiv_bu(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lasx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lasx_xvdiv_hu(a: m256i, b: m256i) -> m256i { + unsafe { transmute(__lasx_xvdiv_hu(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lasx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lasx_xvdiv_wu(a: m256i, b: m256i) -> m256i { + unsafe { transmute(__lasx_xvdiv_wu(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lasx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lasx_xvdiv_du(a: m256i, b: m256i) -> m256i { + unsafe { transmute(__lasx_xvdiv_du(transmute(a), transmute(b))) } +} + #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] @@ -1335,6 +1423,62 @@ pub fn lasx_xvhsubw_du_wu(a: m256i, b: m256i) -> m256i { unsafe { transmute(__lasx_xvhsubw_du_wu(transmute(a), transmute(b))) } } +#[inline] +#[target_feature(enable = "lasx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lasx_xvmod_b(a: m256i, b: m256i) -> m256i { + unsafe { transmute(__lasx_xvmod_b(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lasx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lasx_xvmod_h(a: m256i, b: m256i) -> m256i { + unsafe { transmute(__lasx_xvmod_h(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lasx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lasx_xvmod_w(a: m256i, b: m256i) -> m256i { + unsafe { transmute(__lasx_xvmod_w(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lasx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lasx_xvmod_d(a: m256i, b: m256i) -> m256i { + unsafe { transmute(__lasx_xvmod_d(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lasx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lasx_xvmod_bu(a: m256i, b: m256i) -> m256i { + unsafe { transmute(__lasx_xvmod_bu(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lasx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lasx_xvmod_hu(a: m256i, b: m256i) -> m256i { + unsafe { transmute(__lasx_xvmod_hu(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lasx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lasx_xvmod_wu(a: m256i, b: m256i) -> m256i { + unsafe { transmute(__lasx_xvmod_wu(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lasx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lasx_xvmod_du(a: m256i, b: m256i) -> m256i { + unsafe { transmute(__lasx_xvmod_du(transmute(a), transmute(b))) } +} + #[inline] #[target_feature(enable = "lasx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] diff --git a/library/stdarch/crates/core_arch/src/loongarch64/lasx/portable.rs b/library/stdarch/crates/core_arch/src/loongarch64/lasx/portable.rs index d53f21c792626..fe45ed6543dfb 100644 --- a/library/stdarch/crates/core_arch/src/loongarch64/lasx/portable.rs +++ b/library/stdarch/crates/core_arch/src/loongarch64/lasx/portable.rs @@ -681,22 +681,6 @@ impl_vvv!("lasx", lasx_xvmul_b, simd_mul, m256i, i8x32); impl_vvv!("lasx", lasx_xvmul_h, simd_mul, m256i, i16x16); impl_vvv!("lasx", lasx_xvmul_w, simd_mul, m256i, i32x8); impl_vvv!("lasx", lasx_xvmul_d, simd_mul, m256i, i64x4); -impl_vvv!("lasx", lasx_xvdiv_b, simd_div, m256i, i8x32); -impl_vvv!("lasx", lasx_xvdiv_h, simd_div, m256i, i16x16); -impl_vvv!("lasx", lasx_xvdiv_w, simd_div, m256i, i32x8); -impl_vvv!("lasx", lasx_xvdiv_d, simd_div, m256i, i64x4); -impl_vvv!("lasx", lasx_xvdiv_bu, simd_div, m256i, u8x32); -impl_vvv!("lasx", lasx_xvdiv_hu, simd_div, m256i, u16x16); -impl_vvv!("lasx", lasx_xvdiv_wu, simd_div, m256i, u32x8); -impl_vvv!("lasx", lasx_xvdiv_du, simd_div, m256i, u64x4); -impl_vvv!("lasx", lasx_xvmod_b, simd_rem, m256i, i8x32); -impl_vvv!("lasx", lasx_xvmod_h, simd_rem, m256i, i16x16); -impl_vvv!("lasx", lasx_xvmod_w, simd_rem, m256i, i32x8); -impl_vvv!("lasx", lasx_xvmod_d, simd_rem, m256i, i64x4); -impl_vvv!("lasx", lasx_xvmod_bu, simd_rem, m256i, u8x32); -impl_vvv!("lasx", lasx_xvmod_hu, simd_rem, m256i, u16x16); -impl_vvv!("lasx", lasx_xvmod_wu, simd_rem, m256i, u32x8); -impl_vvv!("lasx", lasx_xvmod_du, simd_rem, m256i, u64x4); impl_vvv!("lasx", lasx_xvand_v, simd_and, m256i, u8x32); impl_vvv!("lasx", lasx_xvandn_v, simd_ext_andn, m256i, u8x32); impl_vvv!("lasx", lasx_xvor_v, simd_or, m256i, u8x32); diff --git a/library/stdarch/crates/core_arch/src/loongarch64/lsx/generated.rs b/library/stdarch/crates/core_arch/src/loongarch64/lsx/generated.rs index aa4e31ce8da6c..601b40467c6b2 100644 --- a/library/stdarch/crates/core_arch/src/loongarch64/lsx/generated.rs +++ b/library/stdarch/crates/core_arch/src/loongarch64/lsx/generated.rs @@ -91,6 +91,22 @@ unsafe extern "llvm-intrinsic" { fn __lsx_vsat_wu(a: __v4u32, b: u32) -> __v4u32; #[link_name = "llvm.loongarch.lsx.vsat.du"] fn __lsx_vsat_du(a: __v2u64, b: u32) -> __v2u64; + #[link_name = "llvm.loongarch.lsx.vdiv.b"] + fn __lsx_vdiv_b(a: __v16i8, b: __v16i8) -> __v16i8; + #[link_name = "llvm.loongarch.lsx.vdiv.h"] + fn __lsx_vdiv_h(a: __v8i16, b: __v8i16) -> __v8i16; + #[link_name = "llvm.loongarch.lsx.vdiv.w"] + fn __lsx_vdiv_w(a: __v4i32, b: __v4i32) -> __v4i32; + #[link_name = "llvm.loongarch.lsx.vdiv.d"] + fn __lsx_vdiv_d(a: __v2i64, b: __v2i64) -> __v2i64; + #[link_name = "llvm.loongarch.lsx.vdiv.bu"] + fn __lsx_vdiv_bu(a: __v16u8, b: __v16u8) -> __v16u8; + #[link_name = "llvm.loongarch.lsx.vdiv.hu"] + fn __lsx_vdiv_hu(a: __v8u16, b: __v8u16) -> __v8u16; + #[link_name = "llvm.loongarch.lsx.vdiv.wu"] + fn __lsx_vdiv_wu(a: __v4u32, b: __v4u32) -> __v4u32; + #[link_name = "llvm.loongarch.lsx.vdiv.du"] + fn __lsx_vdiv_du(a: __v2u64, b: __v2u64) -> __v2u64; #[link_name = "llvm.loongarch.lsx.vhaddw.h.b"] fn __lsx_vhaddw_h_b(a: __v16i8, b: __v16i8) -> __v8i16; #[link_name = "llvm.loongarch.lsx.vhaddw.w.h"] @@ -115,6 +131,22 @@ unsafe extern "llvm-intrinsic" { fn __lsx_vhsubw_wu_hu(a: __v8u16, b: __v8u16) -> __v4i32; #[link_name = "llvm.loongarch.lsx.vhsubw.du.wu"] fn __lsx_vhsubw_du_wu(a: __v4u32, b: __v4u32) -> __v2i64; + #[link_name = "llvm.loongarch.lsx.vmod.b"] + fn __lsx_vmod_b(a: __v16i8, b: __v16i8) -> __v16i8; + #[link_name = "llvm.loongarch.lsx.vmod.h"] + fn __lsx_vmod_h(a: __v8i16, b: __v8i16) -> __v8i16; + #[link_name = "llvm.loongarch.lsx.vmod.w"] + fn __lsx_vmod_w(a: __v4i32, b: __v4i32) -> __v4i32; + #[link_name = "llvm.loongarch.lsx.vmod.d"] + fn __lsx_vmod_d(a: __v2i64, b: __v2i64) -> __v2i64; + #[link_name = "llvm.loongarch.lsx.vmod.bu"] + fn __lsx_vmod_bu(a: __v16u8, b: __v16u8) -> __v16u8; + #[link_name = "llvm.loongarch.lsx.vmod.hu"] + fn __lsx_vmod_hu(a: __v8u16, b: __v8u16) -> __v8u16; + #[link_name = "llvm.loongarch.lsx.vmod.wu"] + fn __lsx_vmod_wu(a: __v4u32, b: __v4u32) -> __v4u32; + #[link_name = "llvm.loongarch.lsx.vmod.du"] + fn __lsx_vmod_du(a: __v2u64, b: __v2u64) -> __v2u64; #[link_name = "llvm.loongarch.lsx.vreplve.b"] fn __lsx_vreplve_b(a: __v16i8, b: i32) -> __v16i8; #[link_name = "llvm.loongarch.lsx.vreplve.h"] @@ -1171,6 +1203,62 @@ pub fn lsx_vsat_du(a: m128i) -> m128i { unsafe { transmute(__lsx_vsat_du(transmute(a), IMM6)) } } +#[inline] +#[target_feature(enable = "lsx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lsx_vdiv_b(a: m128i, b: m128i) -> m128i { + unsafe { transmute(__lsx_vdiv_b(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lsx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lsx_vdiv_h(a: m128i, b: m128i) -> m128i { + unsafe { transmute(__lsx_vdiv_h(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lsx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lsx_vdiv_w(a: m128i, b: m128i) -> m128i { + unsafe { transmute(__lsx_vdiv_w(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lsx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lsx_vdiv_d(a: m128i, b: m128i) -> m128i { + unsafe { transmute(__lsx_vdiv_d(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lsx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lsx_vdiv_bu(a: m128i, b: m128i) -> m128i { + unsafe { transmute(__lsx_vdiv_bu(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lsx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lsx_vdiv_hu(a: m128i, b: m128i) -> m128i { + unsafe { transmute(__lsx_vdiv_hu(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lsx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lsx_vdiv_wu(a: m128i, b: m128i) -> m128i { + unsafe { transmute(__lsx_vdiv_wu(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lsx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lsx_vdiv_du(a: m128i, b: m128i) -> m128i { + unsafe { transmute(__lsx_vdiv_du(transmute(a), transmute(b))) } +} + #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] @@ -1255,6 +1343,62 @@ pub fn lsx_vhsubw_du_wu(a: m128i, b: m128i) -> m128i { unsafe { transmute(__lsx_vhsubw_du_wu(transmute(a), transmute(b))) } } +#[inline] +#[target_feature(enable = "lsx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lsx_vmod_b(a: m128i, b: m128i) -> m128i { + unsafe { transmute(__lsx_vmod_b(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lsx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lsx_vmod_h(a: m128i, b: m128i) -> m128i { + unsafe { transmute(__lsx_vmod_h(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lsx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lsx_vmod_w(a: m128i, b: m128i) -> m128i { + unsafe { transmute(__lsx_vmod_w(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lsx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lsx_vmod_d(a: m128i, b: m128i) -> m128i { + unsafe { transmute(__lsx_vmod_d(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lsx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lsx_vmod_bu(a: m128i, b: m128i) -> m128i { + unsafe { transmute(__lsx_vmod_bu(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lsx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lsx_vmod_hu(a: m128i, b: m128i) -> m128i { + unsafe { transmute(__lsx_vmod_hu(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lsx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lsx_vmod_wu(a: m128i, b: m128i) -> m128i { + unsafe { transmute(__lsx_vmod_wu(transmute(a), transmute(b))) } +} + +#[inline] +#[target_feature(enable = "lsx")] +#[unstable(feature = "stdarch_loongarch", issue = "117427")] +pub fn lsx_vmod_du(a: m128i, b: m128i) -> m128i { + unsafe { transmute(__lsx_vmod_du(transmute(a), transmute(b))) } +} + #[inline] #[target_feature(enable = "lsx")] #[unstable(feature = "stdarch_loongarch", issue = "117427")] diff --git a/library/stdarch/crates/core_arch/src/loongarch64/lsx/portable.rs b/library/stdarch/crates/core_arch/src/loongarch64/lsx/portable.rs index 2b5bfe1ab4d2b..f2b86dae00af8 100644 --- a/library/stdarch/crates/core_arch/src/loongarch64/lsx/portable.rs +++ b/library/stdarch/crates/core_arch/src/loongarch64/lsx/portable.rs @@ -351,22 +351,6 @@ impl_vvv!("lsx", lsx_vmul_b, simd_mul, m128i, i8x16); impl_vvv!("lsx", lsx_vmul_h, simd_mul, m128i, i16x8); impl_vvv!("lsx", lsx_vmul_w, simd_mul, m128i, i32x4); impl_vvv!("lsx", lsx_vmul_d, simd_mul, m128i, i64x2); -impl_vvv!("lsx", lsx_vdiv_b, simd_div, m128i, i8x16); -impl_vvv!("lsx", lsx_vdiv_h, simd_div, m128i, i16x8); -impl_vvv!("lsx", lsx_vdiv_w, simd_div, m128i, i32x4); -impl_vvv!("lsx", lsx_vdiv_d, simd_div, m128i, i64x2); -impl_vvv!("lsx", lsx_vdiv_bu, simd_div, m128i, u8x16); -impl_vvv!("lsx", lsx_vdiv_hu, simd_div, m128i, u16x8); -impl_vvv!("lsx", lsx_vdiv_wu, simd_div, m128i, u32x4); -impl_vvv!("lsx", lsx_vdiv_du, simd_div, m128i, u64x2); -impl_vvv!("lsx", lsx_vmod_b, simd_rem, m128i, i8x16); -impl_vvv!("lsx", lsx_vmod_h, simd_rem, m128i, i16x8); -impl_vvv!("lsx", lsx_vmod_w, simd_rem, m128i, i32x4); -impl_vvv!("lsx", lsx_vmod_d, simd_rem, m128i, i64x2); -impl_vvv!("lsx", lsx_vmod_bu, simd_rem, m128i, u8x16); -impl_vvv!("lsx", lsx_vmod_hu, simd_rem, m128i, u16x8); -impl_vvv!("lsx", lsx_vmod_wu, simd_rem, m128i, u32x4); -impl_vvv!("lsx", lsx_vmod_du, simd_rem, m128i, u64x2); impl_vvv!("lsx", lsx_vand_v, simd_and, m128i, u8x16); impl_vvv!("lsx", lsx_vandn_v, simd_ext_andn, m128i, u8x16); impl_vvv!("lsx", lsx_vor_v, simd_or, m128i, u8x16); diff --git a/library/stdarch/crates/core_arch/src/nvptx/packed.rs b/library/stdarch/crates/core_arch/src/nvptx/packed.rs index 4933a601231f5..99220038e23a2 100644 --- a/library/stdarch/crates/core_arch/src/nvptx/packed.rs +++ b/library/stdarch/crates/core_arch/src/nvptx/packed.rs @@ -30,7 +30,7 @@ types! { /// - [`__hadd2`](https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH____HALF2__ARITHMETIC.html#group__CUDA__MATH____HALF2__ARITHMETIC_1g921c795176eaa31265bd80ef4fe4b8e6) /// - [`__hadd2_rn`](https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH____HALF2__ARITHMETIC.html#group__CUDA__MATH____HALF2__ARITHMETIC_1g6cd8ddb2c3d670e1a10c3eb2e7644f82) #[inline] -#[cfg_attr(test, assert_instr(add.rn.f16x22))] +#[cfg_attr(test, assert_instr(add.rn.f16x2))] #[unstable(feature = "stdarch_nvptx", issue = "111199")] pub unsafe fn f16x2_add(a: f16x2, b: f16x2) -> f16x2 { simd_add(a, b) @@ -68,9 +68,7 @@ pub unsafe fn f16x2_mul(a: f16x2, b: f16x2) -> f16x2 { /// /// /// -/// Corresponds to the CUDA C intrinsics: -/// - [`__fma2`](https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH____HALF2__ARITHMETIC.html#group__CUDA__MATH____HALF2__ARITHMETIC_1g43628ba21ded8b1e188a367348008dab) -/// - [`__fma2_rn`](https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH____HALF2__ARITHMETIC.html#group__CUDA__MATH____HALF2__ARITHMETIC_1g43628ba21ded8b1e188a367348008dab) +/// Corresponds to the CUDA C intrinsic [`__hfma2`](https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH____HALF2__ARITHMETIC.html#group__CUDA__MATH____HALF2__ARITHMETIC_1g43628ba21ded8b1e188a367348008dab) #[inline] #[cfg_attr(test, assert_instr(fma.rn.f16x2))] #[unstable(feature = "stdarch_nvptx", issue = "111199")] @@ -82,7 +80,7 @@ pub unsafe fn f16x2_fma(a: f16x2, b: f16x2, c: f16x2) -> f16x2 { /// /// /// -/// Corresponds to the CUDA C intrinsic [`__hmin2`](https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH____HALF2__COMPARISON.html#group__CUDA__MATH____HALF2__COMPARISON_1g9e17a33f96061804166f3fbd395422b6) +/// Corresponds to the CUDA C intrinsic [`__hneg2`](https://docs.nvidia.com/cuda/cuda-math-api/group__CUDA__MATH____HALF2__ARITHMETIC.html#group__CUDA__MATH____HALF2__ARITHMETIC_1g90e216094729014f541fa1e5a5aaeaf7) #[inline] #[cfg_attr(test, assert_instr(neg.f16x2))] #[unstable(feature = "stdarch_nvptx", issue = "111199")] diff --git a/library/stdarch/crates/core_arch/src/x86/avx2.rs b/library/stdarch/crates/core_arch/src/x86/avx2.rs index 7ad45421cf89f..168871188868b 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx2.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx2.rs @@ -290,14 +290,8 @@ pub const fn _mm256_andnot_si256(a: __m256i, b: __m256i) -> __m256i { #[target_feature(enable = "avx2")] #[cfg_attr(test, assert_instr(vpavgw))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_avg_epu16(a: __m256i, b: __m256i) -> __m256i { - unsafe { - let a = simd_cast::<_, u32x16>(a.as_u16x16()); - let b = simd_cast::<_, u32x16>(b.as_u16x16()); - let r = simd_shr(simd_add(simd_add(a, b), u32x16::splat(1)), u32x16::splat(1)); - transmute(simd_cast::<_, u16x16>(r)) - } +pub fn _mm256_avg_epu16(a: __m256i, b: __m256i) -> __m256i { + unsafe { transmute(pavgw(a.as_u16x16(), b.as_u16x16())) } } /// Averages packed unsigned 8-bit integers in `a` and `b`. @@ -307,14 +301,8 @@ pub const fn _mm256_avg_epu16(a: __m256i, b: __m256i) -> __m256i { #[target_feature(enable = "avx2")] #[cfg_attr(test, assert_instr(vpavgb))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_avg_epu8(a: __m256i, b: __m256i) -> __m256i { - unsafe { - let a = simd_cast::<_, u16x32>(a.as_u8x32()); - let b = simd_cast::<_, u16x32>(b.as_u8x32()); - let r = simd_shr(simd_add(simd_add(a, b), u16x32::splat(1)), u16x32::splat(1)); - transmute(simd_cast::<_, u8x32>(r)) - } +pub fn _mm256_avg_epu8(a: __m256i, b: __m256i) -> __m256i { + unsafe { transmute(pavgb(a.as_u8x32(), b.as_u8x32())) } } /// Blends packed 32-bit integers from `a` and `b` using control mask `IMM4`. @@ -2231,14 +2219,8 @@ pub const fn _mm256_mul_epu32(a: __m256i, b: __m256i) -> __m256i { #[target_feature(enable = "avx2")] #[cfg_attr(test, assert_instr(vpmulhw))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_mulhi_epi16(a: __m256i, b: __m256i) -> __m256i { - unsafe { - let a = simd_cast::<_, i32x16>(a.as_i16x16()); - let b = simd_cast::<_, i32x16>(b.as_i16x16()); - let r = simd_shr(simd_mul(a, b), i32x16::splat(16)); - transmute(simd_cast::(r)) - } +pub fn _mm256_mulhi_epi16(a: __m256i, b: __m256i) -> __m256i { + unsafe { transmute(pmulhw(a.as_i16x16(), b.as_i16x16())) } } /// Multiplies the packed unsigned 16-bit integers in `a` and `b`, producing @@ -2250,14 +2232,8 @@ pub const fn _mm256_mulhi_epi16(a: __m256i, b: __m256i) -> __m256i { #[target_feature(enable = "avx2")] #[cfg_attr(test, assert_instr(vpmulhuw))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_mulhi_epu16(a: __m256i, b: __m256i) -> __m256i { - unsafe { - let a = simd_cast::<_, u32x16>(a.as_u16x16()); - let b = simd_cast::<_, u32x16>(b.as_u16x16()); - let r = simd_shr(simd_mul(a, b), u32x16::splat(16)); - transmute(simd_cast::(r)) - } +pub fn _mm256_mulhi_epu16(a: __m256i, b: __m256i) -> __m256i { + unsafe { transmute(pmulhuw(a.as_u16x16(), b.as_u16x16())) } } /// Multiplies the packed 16-bit integers in `a` and `b`, producing @@ -3759,12 +3735,20 @@ pub const fn _mm256_extract_epi16(a: __m256i) -> i32 { #[allow(improper_ctypes)] unsafe extern "llvm-intrinsic" { + #[link_name = "llvm.x86.avx2.pavg.b"] + fn pavgb(a: u8x32, b: u8x32) -> u8x32; + #[link_name = "llvm.x86.avx2.pavg.w"] + fn pavgw(a: u16x16, b: u16x16) -> u16x16; #[link_name = "llvm.x86.avx2.pmadd.wd"] fn pmaddwd(a: i16x16, b: i16x16) -> i32x8; #[link_name = "llvm.x86.avx2.pmadd.ub.sw"] fn pmaddubsw(a: u8x32, b: i8x32) -> i16x16; #[link_name = "llvm.x86.avx2.mpsadbw"] fn mpsadbw(a: u8x32, b: u8x32, imm8: i8) -> u16x16; + #[link_name = "llvm.x86.avx2.pmulhu.w"] + fn pmulhuw(a: u16x16, b: u16x16) -> u16x16; + #[link_name = "llvm.x86.avx2.pmulh.w"] + fn pmulhw(a: i16x16, b: i16x16) -> i16x16; #[link_name = "llvm.x86.avx2.pmul.hr.sw"] fn pmulhrsw(a: i16x16, b: i16x16) -> i16x16; #[link_name = "llvm.x86.avx2.packsswb"] @@ -4186,14 +4170,14 @@ mod tests { } #[simd_test(enable = "avx2")] - const fn test_mm256_avg_epu8() { + fn test_mm256_avg_epu8() { let (a, b) = (_mm256_set1_epi8(3), _mm256_set1_epi8(9)); let r = _mm256_avg_epu8(a, b); assert_eq_m256i(r, _mm256_set1_epi8(6)); } #[simd_test(enable = "avx2")] - const fn test_mm256_avg_epu16() { + fn test_mm256_avg_epu16() { let (a, b) = (_mm256_set1_epi16(3), _mm256_set1_epi16(9)); let r = _mm256_avg_epu16(a, b); assert_eq_m256i(r, _mm256_set1_epi16(6)); @@ -4895,7 +4879,7 @@ mod tests { } #[simd_test(enable = "avx2")] - const fn test_mm256_mulhi_epi16() { + fn test_mm256_mulhi_epi16() { let a = _mm256_set1_epi16(6535); let b = _mm256_set1_epi16(6535); let r = _mm256_mulhi_epi16(a, b); @@ -4904,7 +4888,7 @@ mod tests { } #[simd_test(enable = "avx2")] - const fn test_mm256_mulhi_epu16() { + fn test_mm256_mulhi_epu16() { let a = _mm256_set1_epi16(6535); let b = _mm256_set1_epi16(6535); let r = _mm256_mulhi_epu16(a, b); diff --git a/library/stdarch/crates/core_arch/src/x86/avx512bw.rs b/library/stdarch/crates/core_arch/src/x86/avx512bw.rs index e400453f627bc..bc119870fa99a 100644 --- a/library/stdarch/crates/core_arch/src/x86/avx512bw.rs +++ b/library/stdarch/crates/core_arch/src/x86/avx512bw.rs @@ -1450,14 +1450,8 @@ pub const fn _mm_maskz_subs_epi8(k: __mmask16, a: __m128i, b: __m128i) -> __m128 #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpmulhuw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_mulhi_epu16(a: __m512i, b: __m512i) -> __m512i { - unsafe { - let a = simd_cast::<_, u32x32>(a.as_u16x32()); - let b = simd_cast::<_, u32x32>(b.as_u16x32()); - let r = simd_shr(simd_mul(a, b), u32x32::splat(16)); - transmute(simd_cast::(r)) - } +pub fn _mm512_mulhi_epu16(a: __m512i, b: __m512i) -> __m512i { + unsafe { transmute(vpmulhuw(a.as_u16x32(), b.as_u16x32())) } } /// Multiply the packed unsigned 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -1467,13 +1461,7 @@ pub const fn _mm512_mulhi_epu16(a: __m512i, b: __m512i) -> __m512i { #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpmulhuw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_mask_mulhi_epu16( - src: __m512i, - k: __mmask32, - a: __m512i, - b: __m512i, -) -> __m512i { +pub fn _mm512_mask_mulhi_epu16(src: __m512i, k: __mmask32, a: __m512i, b: __m512i) -> __m512i { unsafe { let mul = _mm512_mulhi_epu16(a, b).as_u16x32(); transmute(simd_select_bitmask(k, mul, src.as_u16x32())) @@ -1487,8 +1475,7 @@ pub const fn _mm512_mask_mulhi_epu16( #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpmulhuw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_maskz_mulhi_epu16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { +pub fn _mm512_maskz_mulhi_epu16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { unsafe { let mul = _mm512_mulhi_epu16(a, b).as_u16x32(); transmute(simd_select_bitmask(k, mul, u16x32::ZERO)) @@ -1502,13 +1489,7 @@ pub const fn _mm512_maskz_mulhi_epu16(k: __mmask32, a: __m512i, b: __m512i) -> _ #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpmulhuw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_mask_mulhi_epu16( - src: __m256i, - k: __mmask16, - a: __m256i, - b: __m256i, -) -> __m256i { +pub fn _mm256_mask_mulhi_epu16(src: __m256i, k: __mmask16, a: __m256i, b: __m256i) -> __m256i { unsafe { let mul = _mm256_mulhi_epu16(a, b).as_u16x16(); transmute(simd_select_bitmask(k, mul, src.as_u16x16())) @@ -1522,8 +1503,7 @@ pub const fn _mm256_mask_mulhi_epu16( #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpmulhuw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_maskz_mulhi_epu16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { +pub fn _mm256_maskz_mulhi_epu16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { unsafe { let mul = _mm256_mulhi_epu16(a, b).as_u16x16(); transmute(simd_select_bitmask(k, mul, u16x16::ZERO)) @@ -1537,8 +1517,7 @@ pub const fn _mm256_maskz_mulhi_epu16(k: __mmask16, a: __m256i, b: __m256i) -> _ #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpmulhuw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_mask_mulhi_epu16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { +pub fn _mm_mask_mulhi_epu16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { unsafe { let mul = _mm_mulhi_epu16(a, b).as_u16x8(); transmute(simd_select_bitmask(k, mul, src.as_u16x8())) @@ -1552,8 +1531,7 @@ pub const fn _mm_mask_mulhi_epu16(src: __m128i, k: __mmask8, a: __m128i, b: __m1 #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpmulhuw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_maskz_mulhi_epu16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { +pub fn _mm_maskz_mulhi_epu16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { unsafe { let mul = _mm_mulhi_epu16(a, b).as_u16x8(); transmute(simd_select_bitmask(k, mul, u16x8::ZERO)) @@ -1567,14 +1545,8 @@ pub const fn _mm_maskz_mulhi_epu16(k: __mmask8, a: __m128i, b: __m128i) -> __m12 #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpmulhw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_mulhi_epi16(a: __m512i, b: __m512i) -> __m512i { - unsafe { - let a = simd_cast::<_, i32x32>(a.as_i16x32()); - let b = simd_cast::<_, i32x32>(b.as_i16x32()); - let r = simd_shr(simd_mul(a, b), i32x32::splat(16)); - transmute(simd_cast::(r)) - } +pub fn _mm512_mulhi_epi16(a: __m512i, b: __m512i) -> __m512i { + unsafe { transmute(vpmulhw(a.as_i16x32(), b.as_i16x32())) } } /// Multiply the packed signed 16-bit integers in a and b, producing intermediate 32-bit integers, and store the high 16 bits of the intermediate integers in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -1584,13 +1556,7 @@ pub const fn _mm512_mulhi_epi16(a: __m512i, b: __m512i) -> __m512i { #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpmulhw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_mask_mulhi_epi16( - src: __m512i, - k: __mmask32, - a: __m512i, - b: __m512i, -) -> __m512i { +pub fn _mm512_mask_mulhi_epi16(src: __m512i, k: __mmask32, a: __m512i, b: __m512i) -> __m512i { unsafe { let mul = _mm512_mulhi_epi16(a, b).as_i16x32(); transmute(simd_select_bitmask(k, mul, src.as_i16x32())) @@ -1604,8 +1570,7 @@ pub const fn _mm512_mask_mulhi_epi16( #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpmulhw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_maskz_mulhi_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { +pub fn _mm512_maskz_mulhi_epi16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { unsafe { let mul = _mm512_mulhi_epi16(a, b).as_i16x32(); transmute(simd_select_bitmask(k, mul, i16x32::ZERO)) @@ -1619,13 +1584,7 @@ pub const fn _mm512_maskz_mulhi_epi16(k: __mmask32, a: __m512i, b: __m512i) -> _ #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpmulhw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_mask_mulhi_epi16( - src: __m256i, - k: __mmask16, - a: __m256i, - b: __m256i, -) -> __m256i { +pub fn _mm256_mask_mulhi_epi16(src: __m256i, k: __mmask16, a: __m256i, b: __m256i) -> __m256i { unsafe { let mul = _mm256_mulhi_epi16(a, b).as_i16x16(); transmute(simd_select_bitmask(k, mul, src.as_i16x16())) @@ -1639,8 +1598,7 @@ pub const fn _mm256_mask_mulhi_epi16( #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpmulhw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_maskz_mulhi_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { +pub fn _mm256_maskz_mulhi_epi16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { unsafe { let mul = _mm256_mulhi_epi16(a, b).as_i16x16(); transmute(simd_select_bitmask(k, mul, i16x16::ZERO)) @@ -1654,8 +1612,7 @@ pub const fn _mm256_maskz_mulhi_epi16(k: __mmask16, a: __m256i, b: __m256i) -> _ #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpmulhw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_mask_mulhi_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { +pub fn _mm_mask_mulhi_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { unsafe { let mul = _mm_mulhi_epi16(a, b).as_i16x8(); transmute(simd_select_bitmask(k, mul, src.as_i16x8())) @@ -1669,8 +1626,7 @@ pub const fn _mm_mask_mulhi_epi16(src: __m128i, k: __mmask8, a: __m128i, b: __m1 #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpmulhw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_maskz_mulhi_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { +pub fn _mm_maskz_mulhi_epi16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { unsafe { let mul = _mm_mulhi_epi16(a, b).as_i16x8(); transmute(simd_select_bitmask(k, mul, i16x8::ZERO)) @@ -6903,14 +6859,8 @@ pub fn _mm_maskz_packus_epi16(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpavgw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_avg_epu16(a: __m512i, b: __m512i) -> __m512i { - unsafe { - let a = simd_cast::<_, u32x32>(a.as_u16x32()); - let b = simd_cast::<_, u32x32>(b.as_u16x32()); - let r = simd_shr(simd_add(simd_add(a, b), u32x32::splat(1)), u32x32::splat(1)); - transmute(simd_cast::<_, u16x32>(r)) - } +pub fn _mm512_avg_epu16(a: __m512i, b: __m512i) -> __m512i { + unsafe { transmute(vpavgw(a.as_u16x32(), b.as_u16x32())) } } /// Average packed unsigned 16-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -6920,8 +6870,7 @@ pub const fn _mm512_avg_epu16(a: __m512i, b: __m512i) -> __m512i { #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpavgw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_mask_avg_epu16(src: __m512i, k: __mmask32, a: __m512i, b: __m512i) -> __m512i { +pub fn _mm512_mask_avg_epu16(src: __m512i, k: __mmask32, a: __m512i, b: __m512i) -> __m512i { unsafe { let avg = _mm512_avg_epu16(a, b).as_u16x32(); transmute(simd_select_bitmask(k, avg, src.as_u16x32())) @@ -6935,8 +6884,7 @@ pub const fn _mm512_mask_avg_epu16(src: __m512i, k: __mmask32, a: __m512i, b: __ #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpavgw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_maskz_avg_epu16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { +pub fn _mm512_maskz_avg_epu16(k: __mmask32, a: __m512i, b: __m512i) -> __m512i { unsafe { let avg = _mm512_avg_epu16(a, b).as_u16x32(); transmute(simd_select_bitmask(k, avg, u16x32::ZERO)) @@ -6950,8 +6898,7 @@ pub const fn _mm512_maskz_avg_epu16(k: __mmask32, a: __m512i, b: __m512i) -> __m #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpavgw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_mask_avg_epu16(src: __m256i, k: __mmask16, a: __m256i, b: __m256i) -> __m256i { +pub fn _mm256_mask_avg_epu16(src: __m256i, k: __mmask16, a: __m256i, b: __m256i) -> __m256i { unsafe { let avg = _mm256_avg_epu16(a, b).as_u16x16(); transmute(simd_select_bitmask(k, avg, src.as_u16x16())) @@ -6965,8 +6912,7 @@ pub const fn _mm256_mask_avg_epu16(src: __m256i, k: __mmask16, a: __m256i, b: __ #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpavgw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_maskz_avg_epu16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { +pub fn _mm256_maskz_avg_epu16(k: __mmask16, a: __m256i, b: __m256i) -> __m256i { unsafe { let avg = _mm256_avg_epu16(a, b).as_u16x16(); transmute(simd_select_bitmask(k, avg, u16x16::ZERO)) @@ -6980,8 +6926,7 @@ pub const fn _mm256_maskz_avg_epu16(k: __mmask16, a: __m256i, b: __m256i) -> __m #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpavgw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_mask_avg_epu16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { +pub fn _mm_mask_avg_epu16(src: __m128i, k: __mmask8, a: __m128i, b: __m128i) -> __m128i { unsafe { let avg = _mm_avg_epu16(a, b).as_u16x8(); transmute(simd_select_bitmask(k, avg, src.as_u16x8())) @@ -6995,8 +6940,7 @@ pub const fn _mm_mask_avg_epu16(src: __m128i, k: __mmask8, a: __m128i, b: __m128 #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpavgw))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_maskz_avg_epu16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { +pub fn _mm_maskz_avg_epu16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i { unsafe { let avg = _mm_avg_epu16(a, b).as_u16x8(); transmute(simd_select_bitmask(k, avg, u16x8::ZERO)) @@ -7010,14 +6954,8 @@ pub const fn _mm_maskz_avg_epu16(k: __mmask8, a: __m128i, b: __m128i) -> __m128i #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpavgb))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_avg_epu8(a: __m512i, b: __m512i) -> __m512i { - unsafe { - let a = simd_cast::<_, u16x64>(a.as_u8x64()); - let b = simd_cast::<_, u16x64>(b.as_u8x64()); - let r = simd_shr(simd_add(simd_add(a, b), u16x64::splat(1)), u16x64::splat(1)); - transmute(simd_cast::<_, u8x64>(r)) - } +pub fn _mm512_avg_epu8(a: __m512i, b: __m512i) -> __m512i { + unsafe { transmute(vpavgb(a.as_u8x64(), b.as_u8x64())) } } /// Average packed unsigned 8-bit integers in a and b, and store the results in dst using writemask k (elements are copied from src when the corresponding mask bit is not set). @@ -7027,8 +6965,7 @@ pub const fn _mm512_avg_epu8(a: __m512i, b: __m512i) -> __m512i { #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpavgb))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_mask_avg_epu8(src: __m512i, k: __mmask64, a: __m512i, b: __m512i) -> __m512i { +pub fn _mm512_mask_avg_epu8(src: __m512i, k: __mmask64, a: __m512i, b: __m512i) -> __m512i { unsafe { let avg = _mm512_avg_epu8(a, b).as_u8x64(); transmute(simd_select_bitmask(k, avg, src.as_u8x64())) @@ -7042,8 +6979,7 @@ pub const fn _mm512_mask_avg_epu8(src: __m512i, k: __mmask64, a: __m512i, b: __m #[target_feature(enable = "avx512bw")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpavgb))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm512_maskz_avg_epu8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { +pub fn _mm512_maskz_avg_epu8(k: __mmask64, a: __m512i, b: __m512i) -> __m512i { unsafe { let avg = _mm512_avg_epu8(a, b).as_u8x64(); transmute(simd_select_bitmask(k, avg, u8x64::ZERO)) @@ -7057,8 +6993,7 @@ pub const fn _mm512_maskz_avg_epu8(k: __mmask64, a: __m512i, b: __m512i) -> __m5 #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpavgb))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_mask_avg_epu8(src: __m256i, k: __mmask32, a: __m256i, b: __m256i) -> __m256i { +pub fn _mm256_mask_avg_epu8(src: __m256i, k: __mmask32, a: __m256i, b: __m256i) -> __m256i { unsafe { let avg = _mm256_avg_epu8(a, b).as_u8x32(); transmute(simd_select_bitmask(k, avg, src.as_u8x32())) @@ -7072,8 +7007,7 @@ pub const fn _mm256_mask_avg_epu8(src: __m256i, k: __mmask32, a: __m256i, b: __m #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpavgb))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm256_maskz_avg_epu8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { +pub fn _mm256_maskz_avg_epu8(k: __mmask32, a: __m256i, b: __m256i) -> __m256i { unsafe { let avg = _mm256_avg_epu8(a, b).as_u8x32(); transmute(simd_select_bitmask(k, avg, u8x32::ZERO)) @@ -7087,8 +7021,7 @@ pub const fn _mm256_maskz_avg_epu8(k: __mmask32, a: __m256i, b: __m256i) -> __m2 #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpavgb))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_mask_avg_epu8(src: __m128i, k: __mmask16, a: __m128i, b: __m128i) -> __m128i { +pub fn _mm_mask_avg_epu8(src: __m128i, k: __mmask16, a: __m128i, b: __m128i) -> __m128i { unsafe { let avg = _mm_avg_epu8(a, b).as_u8x16(); transmute(simd_select_bitmask(k, avg, src.as_u8x16())) @@ -7102,8 +7035,7 @@ pub const fn _mm_mask_avg_epu8(src: __m128i, k: __mmask16, a: __m128i, b: __m128 #[target_feature(enable = "avx512bw,avx512vl")] #[stable(feature = "stdarch_x86_avx512", since = "1.89")] #[cfg_attr(test, assert_instr(vpavgb))] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_maskz_avg_epu8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { +pub fn _mm_maskz_avg_epu8(k: __mmask16, a: __m128i, b: __m128i) -> __m128i { unsafe { let avg = _mm_avg_epu8(a, b).as_u8x16(); transmute(simd_select_bitmask(k, avg, u8x16::ZERO)) @@ -12499,6 +12431,11 @@ unsafe extern "llvm-intrinsic" { #[link_name = "llvm.x86.avx512.pmul.hr.sw.512"] fn vpmulhrsw(a: i16x32, b: i16x32) -> i16x32; + #[link_name = "llvm.x86.avx512.pmulhu.w.512"] + fn vpmulhuw(a: u16x32, b: u16x32) -> u16x32; + #[link_name = "llvm.x86.avx512.pmulh.w.512"] + fn vpmulhw(a: i16x32, b: i16x32) -> i16x32; + #[link_name = "llvm.x86.avx512.pmaddw.d.512"] fn vpmaddwd(a: i16x32, b: i16x32) -> i32x16; #[link_name = "llvm.x86.avx512.pmaddubs.w.512"] @@ -12581,6 +12518,11 @@ unsafe extern "llvm-intrinsic" { #[link_name = "llvm.x86.avx512.mask.pmovus.wb.mem.512"] fn vpmovuswbmem(mem_addr: *mut i8, a: i16x32, mask: u32); + + #[link_name = "llvm.x86.avx512.pavg.b.512"] + fn vpavgb(a: u8x64, b: u8x64) -> u8x64; + #[link_name = "llvm.x86.avx512.pavg.w.512"] + fn vpavgw(a: u16x32, b: u16x32) -> u16x32; } #[cfg(test)] @@ -13818,7 +13760,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_mulhi_epu16() { + fn test_mm512_mulhi_epu16() { let a = _mm512_set1_epi16(1); let b = _mm512_set1_epi16(1); let r = _mm512_mulhi_epu16(a, b); @@ -13827,7 +13769,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_mask_mulhi_epu16() { + fn test_mm512_mask_mulhi_epu16() { let a = _mm512_set1_epi16(1); let b = _mm512_set1_epi16(1); let r = _mm512_mask_mulhi_epu16(a, 0, a, b); @@ -13840,7 +13782,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_maskz_mulhi_epu16() { + fn test_mm512_maskz_mulhi_epu16() { let a = _mm512_set1_epi16(1); let b = _mm512_set1_epi16(1); let r = _mm512_maskz_mulhi_epu16(0, a, b); @@ -13853,7 +13795,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_mask_mulhi_epu16() { + fn test_mm256_mask_mulhi_epu16() { let a = _mm256_set1_epi16(1); let b = _mm256_set1_epi16(1); let r = _mm256_mask_mulhi_epu16(a, 0, a, b); @@ -13864,7 +13806,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_maskz_mulhi_epu16() { + fn test_mm256_maskz_mulhi_epu16() { let a = _mm256_set1_epi16(1); let b = _mm256_set1_epi16(1); let r = _mm256_maskz_mulhi_epu16(0, a, b); @@ -13875,7 +13817,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_mask_mulhi_epu16() { + fn test_mm_mask_mulhi_epu16() { let a = _mm_set1_epi16(1); let b = _mm_set1_epi16(1); let r = _mm_mask_mulhi_epu16(a, 0, a, b); @@ -13886,7 +13828,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_maskz_mulhi_epu16() { + fn test_mm_maskz_mulhi_epu16() { let a = _mm_set1_epi16(1); let b = _mm_set1_epi16(1); let r = _mm_maskz_mulhi_epu16(0, a, b); @@ -13897,7 +13839,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_mulhi_epi16() { + fn test_mm512_mulhi_epi16() { let a = _mm512_set1_epi16(1); let b = _mm512_set1_epi16(1); let r = _mm512_mulhi_epi16(a, b); @@ -13906,7 +13848,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_mask_mulhi_epi16() { + fn test_mm512_mask_mulhi_epi16() { let a = _mm512_set1_epi16(1); let b = _mm512_set1_epi16(1); let r = _mm512_mask_mulhi_epi16(a, 0, a, b); @@ -13919,7 +13861,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_maskz_mulhi_epi16() { + fn test_mm512_maskz_mulhi_epi16() { let a = _mm512_set1_epi16(1); let b = _mm512_set1_epi16(1); let r = _mm512_maskz_mulhi_epi16(0, a, b); @@ -13932,7 +13874,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_mask_mulhi_epi16() { + fn test_mm256_mask_mulhi_epi16() { let a = _mm256_set1_epi16(1); let b = _mm256_set1_epi16(1); let r = _mm256_mask_mulhi_epi16(a, 0, a, b); @@ -13943,7 +13885,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_maskz_mulhi_epi16() { + fn test_mm256_maskz_mulhi_epi16() { let a = _mm256_set1_epi16(1); let b = _mm256_set1_epi16(1); let r = _mm256_maskz_mulhi_epi16(0, a, b); @@ -13954,7 +13896,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_mask_mulhi_epi16() { + fn test_mm_mask_mulhi_epi16() { let a = _mm_set1_epi16(1); let b = _mm_set1_epi16(1); let r = _mm_mask_mulhi_epi16(a, 0, a, b); @@ -13965,7 +13907,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_maskz_mulhi_epi16() { + fn test_mm_maskz_mulhi_epi16() { let a = _mm_set1_epi16(1); let b = _mm_set1_epi16(1); let r = _mm_maskz_mulhi_epi16(0, a, b); @@ -17987,7 +17929,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_avg_epu16() { + fn test_mm512_avg_epu16() { let a = _mm512_set1_epi16(1); let b = _mm512_set1_epi16(1); let r = _mm512_avg_epu16(a, b); @@ -17996,7 +17938,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_mask_avg_epu16() { + fn test_mm512_mask_avg_epu16() { let a = _mm512_set1_epi16(1); let b = _mm512_set1_epi16(1); let r = _mm512_mask_avg_epu16(a, 0, a, b); @@ -18009,7 +17951,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_maskz_avg_epu16() { + fn test_mm512_maskz_avg_epu16() { let a = _mm512_set1_epi16(1); let b = _mm512_set1_epi16(1); let r = _mm512_maskz_avg_epu16(0, a, b); @@ -18022,7 +17964,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_mask_avg_epu16() { + fn test_mm256_mask_avg_epu16() { let a = _mm256_set1_epi16(1); let b = _mm256_set1_epi16(1); let r = _mm256_mask_avg_epu16(a, 0, a, b); @@ -18033,7 +17975,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_maskz_avg_epu16() { + fn test_mm256_maskz_avg_epu16() { let a = _mm256_set1_epi16(1); let b = _mm256_set1_epi16(1); let r = _mm256_maskz_avg_epu16(0, a, b); @@ -18044,7 +17986,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_mask_avg_epu16() { + fn test_mm_mask_avg_epu16() { let a = _mm_set1_epi16(1); let b = _mm_set1_epi16(1); let r = _mm_mask_avg_epu16(a, 0, a, b); @@ -18055,7 +17997,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_maskz_avg_epu16() { + fn test_mm_maskz_avg_epu16() { let a = _mm_set1_epi16(1); let b = _mm_set1_epi16(1); let r = _mm_maskz_avg_epu16(0, a, b); @@ -18066,7 +18008,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_avg_epu8() { + fn test_mm512_avg_epu8() { let a = _mm512_set1_epi8(1); let b = _mm512_set1_epi8(1); let r = _mm512_avg_epu8(a, b); @@ -18075,7 +18017,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_mask_avg_epu8() { + fn test_mm512_mask_avg_epu8() { let a = _mm512_set1_epi8(1); let b = _mm512_set1_epi8(1); let r = _mm512_mask_avg_epu8(a, 0, a, b); @@ -18095,7 +18037,7 @@ mod tests { } #[simd_test(enable = "avx512bw")] - const fn test_mm512_maskz_avg_epu8() { + fn test_mm512_maskz_avg_epu8() { let a = _mm512_set1_epi8(1); let b = _mm512_set1_epi8(1); let r = _mm512_maskz_avg_epu8(0, a, b); @@ -18114,7 +18056,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_mask_avg_epu8() { + fn test_mm256_mask_avg_epu8() { let a = _mm256_set1_epi8(1); let b = _mm256_set1_epi8(1); let r = _mm256_mask_avg_epu8(a, 0, a, b); @@ -18127,7 +18069,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm256_maskz_avg_epu8() { + fn test_mm256_maskz_avg_epu8() { let a = _mm256_set1_epi8(1); let b = _mm256_set1_epi8(1); let r = _mm256_maskz_avg_epu8(0, a, b); @@ -18140,7 +18082,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_mask_avg_epu8() { + fn test_mm_mask_avg_epu8() { let a = _mm_set1_epi8(1); let b = _mm_set1_epi8(1); let r = _mm_mask_avg_epu8(a, 0, a, b); @@ -18151,7 +18093,7 @@ mod tests { } #[simd_test(enable = "avx512bw,avx512vl")] - const fn test_mm_maskz_avg_epu8() { + fn test_mm_maskz_avg_epu8() { let a = _mm_set1_epi8(1); let b = _mm_set1_epi8(1); let r = _mm_maskz_avg_epu8(0, a, b); diff --git a/library/stdarch/crates/core_arch/src/x86/sse2.rs b/library/stdarch/crates/core_arch/src/x86/sse2.rs index 2c61b095c6b24..b52fce6765a00 100644 --- a/library/stdarch/crates/core_arch/src/x86/sse2.rs +++ b/library/stdarch/crates/core_arch/src/x86/sse2.rs @@ -178,14 +178,8 @@ pub const fn _mm_adds_epu16(a: __m128i, b: __m128i) -> __m128i { #[target_feature(enable = "sse2")] #[cfg_attr(test, assert_instr(pavgb))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_avg_epu8(a: __m128i, b: __m128i) -> __m128i { - unsafe { - let a = simd_cast::<_, u16x16>(a.as_u8x16()); - let b = simd_cast::<_, u16x16>(b.as_u8x16()); - let r = simd_shr(simd_add(simd_add(a, b), u16x16::splat(1)), u16x16::splat(1)); - transmute(simd_cast::<_, u8x16>(r)) - } +pub fn _mm_avg_epu8(a: __m128i, b: __m128i) -> __m128i { + unsafe { transmute(pavgb(a.as_u8x16(), b.as_u8x16())) } } /// Averages packed unsigned 16-bit integers in `a` and `b`. @@ -195,14 +189,8 @@ pub const fn _mm_avg_epu8(a: __m128i, b: __m128i) -> __m128i { #[target_feature(enable = "sse2")] #[cfg_attr(test, assert_instr(pavgw))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_avg_epu16(a: __m128i, b: __m128i) -> __m128i { - unsafe { - let a = simd_cast::<_, u32x8>(a.as_u16x8()); - let b = simd_cast::<_, u32x8>(b.as_u16x8()); - let r = simd_shr(simd_add(simd_add(a, b), u32x8::splat(1)), u32x8::splat(1)); - transmute(simd_cast::<_, u16x8>(r)) - } +pub fn _mm_avg_epu16(a: __m128i, b: __m128i) -> __m128i { + unsafe { transmute(pavgw(a.as_u16x8(), b.as_u16x8())) } } /// Multiplies and then horizontally add signed 16 bit integers in `a` and `b`. @@ -294,14 +282,8 @@ pub const fn _mm_min_epu8(a: __m128i, b: __m128i) -> __m128i { #[target_feature(enable = "sse2")] #[cfg_attr(test, assert_instr(pmulhw))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_mulhi_epi16(a: __m128i, b: __m128i) -> __m128i { - unsafe { - let a = simd_cast::<_, i32x8>(a.as_i16x8()); - let b = simd_cast::<_, i32x8>(b.as_i16x8()); - let r = simd_shr(simd_mul(a, b), i32x8::splat(16)); - transmute(simd_cast::(r)) - } +pub fn _mm_mulhi_epi16(a: __m128i, b: __m128i) -> __m128i { + unsafe { transmute(pmulhw(a.as_i16x8(), b.as_i16x8())) } } /// Multiplies the packed unsigned 16-bit integers in `a` and `b`. @@ -314,14 +296,8 @@ pub const fn _mm_mulhi_epi16(a: __m128i, b: __m128i) -> __m128i { #[target_feature(enable = "sse2")] #[cfg_attr(test, assert_instr(pmulhuw))] #[stable(feature = "simd_x86", since = "1.27.0")] -#[rustc_const_unstable(feature = "stdarch_const_x86", issue = "149298")] -pub const fn _mm_mulhi_epu16(a: __m128i, b: __m128i) -> __m128i { - unsafe { - let a = simd_cast::<_, u32x8>(a.as_u16x8()); - let b = simd_cast::<_, u32x8>(b.as_u16x8()); - let r = simd_shr(simd_mul(a, b), u32x8::splat(16)); - transmute(simd_cast::(r)) - } +pub fn _mm_mulhi_epu16(a: __m128i, b: __m128i) -> __m128i { + unsafe { transmute(pmulhuw(a.as_u16x8(), b.as_u16x8())) } } /// Multiplies the packed 16-bit integers in `a` and `b`. @@ -3200,8 +3176,16 @@ unsafe extern "llvm-intrinsic" { fn lfence(); #[link_name = "llvm.x86.sse2.mfence"] fn mfence(); + #[link_name = "llvm.x86.sse2.pavg.b"] + fn pavgb(a: u8x16, b: u8x16) -> u8x16; + #[link_name = "llvm.x86.sse2.pavg.w"] + fn pavgw(a: u16x8, b: u16x8) -> u16x8; #[link_name = "llvm.x86.sse2.pmadd.wd"] fn pmaddwd(a: i16x8, b: i16x8) -> i32x4; + #[link_name = "llvm.x86.sse2.pmulh.w"] + fn pmulhw(a: i16x8, b: i16x8) -> i16x8; + #[link_name = "llvm.x86.sse2.pmulhu.w"] + fn pmulhuw(a: u16x8, b: u16x8) -> u16x8; #[link_name = "llvm.x86.sse2.psad.bw"] fn psadbw(a: u8x16, b: u8x16) -> u64x2; #[link_name = "llvm.x86.sse2.psll.w"] @@ -3466,14 +3450,14 @@ mod tests { } #[simd_test(enable = "sse2")] - const fn test_mm_avg_epu8() { + fn test_mm_avg_epu8() { let (a, b) = (_mm_set1_epi8(3), _mm_set1_epi8(9)); let r = _mm_avg_epu8(a, b); assert_eq_m128i(r, _mm_set1_epi8(6)); } #[simd_test(enable = "sse2")] - const fn test_mm_avg_epu16() { + fn test_mm_avg_epu16() { let (a, b) = (_mm_set1_epi16(3), _mm_set1_epi16(9)); let r = _mm_avg_epu16(a, b); assert_eq_m128i(r, _mm_set1_epi16(6)); @@ -3547,14 +3531,14 @@ mod tests { } #[simd_test(enable = "sse2")] - const fn test_mm_mulhi_epi16() { + fn test_mm_mulhi_epi16() { let (a, b) = (_mm_set1_epi16(1000), _mm_set1_epi16(-1001)); let r = _mm_mulhi_epi16(a, b); assert_eq_m128i(r, _mm_set1_epi16(-16)); } #[simd_test(enable = "sse2")] - const fn test_mm_mulhi_epu16() { + fn test_mm_mulhi_epu16() { let (a, b) = (_mm_set1_epi16(1000), _mm_set1_epi16(1001)); let r = _mm_mulhi_epu16(a, b); assert_eq_m128i(r, _mm_set1_epi16(15)); diff --git a/library/stdarch/crates/simd-test-macro/src/lib.rs b/library/stdarch/crates/simd-test-macro/src/lib.rs index 9219540a1065f..51dd8aaf46fe7 100644 --- a/library/stdarch/crates/simd-test-macro/src/lib.rs +++ b/library/stdarch/crates/simd-test-macro/src/lib.rs @@ -67,8 +67,7 @@ pub fn simd_test( "arm" | "armv7" | "thumbv7neon" => "is_arm_feature_detected", "aarch64" | "arm64ec" | "aarch64_be" => "is_aarch64_feature_detected", maybe_riscv if maybe_riscv.starts_with("riscv") => "is_riscv_feature_detected", - "powerpc" | "powerpcle" => "is_powerpc_feature_detected", - "powerpc64" | "powerpc64le" => "is_powerpc64_feature_detected", + "powerpc" | "powerpcle" | "powerpc64" | "powerpc64le" => "is_powerpc_feature_detected", "loongarch32" | "loongarch64" => "is_loongarch_feature_detected", "s390x" => "is_s390x_feature_detected", "mips64" | "mips64el" => "is_mips64_feature_detected", diff --git a/library/stdarch/crates/stdarch-gen-loongarch/lasx.spec b/library/stdarch/crates/stdarch-gen-loongarch/lasx.spec index eb3ff074a3978..63895945948f8 100644 --- a/library/stdarch/crates/stdarch-gen-loongarch/lasx.spec +++ b/library/stdarch/crates/stdarch-gen-loongarch/lasx.spec @@ -1260,49 +1260,41 @@ asm-fmts = xd, xj, xk data-types = V4DI, V4DI, V4DI, V4DI /// lasx_xvdiv_b -impl = portable name = lasx_xvdiv_b asm-fmts = xd, xj, xk data-types = V32QI, V32QI, V32QI /// lasx_xvdiv_h -impl = portable name = lasx_xvdiv_h asm-fmts = xd, xj, xk data-types = V16HI, V16HI, V16HI /// lasx_xvdiv_w -impl = portable name = lasx_xvdiv_w asm-fmts = xd, xj, xk data-types = V8SI, V8SI, V8SI /// lasx_xvdiv_d -impl = portable name = lasx_xvdiv_d asm-fmts = xd, xj, xk data-types = V4DI, V4DI, V4DI /// lasx_xvdiv_bu -impl = portable name = lasx_xvdiv_bu asm-fmts = xd, xj, xk data-types = UV32QI, UV32QI, UV32QI /// lasx_xvdiv_hu -impl = portable name = lasx_xvdiv_hu asm-fmts = xd, xj, xk data-types = UV16HI, UV16HI, UV16HI /// lasx_xvdiv_wu -impl = portable name = lasx_xvdiv_wu asm-fmts = xd, xj, xk data-types = UV8SI, UV8SI, UV8SI /// lasx_xvdiv_du -impl = portable name = lasx_xvdiv_du asm-fmts = xd, xj, xk data-types = UV4DI, UV4DI, UV4DI @@ -1368,49 +1360,41 @@ asm-fmts = xd, xj, xk data-types = V4DI, UV8SI, UV8SI /// lasx_xvmod_b -impl = portable name = lasx_xvmod_b asm-fmts = xd, xj, xk data-types = V32QI, V32QI, V32QI /// lasx_xvmod_h -impl = portable name = lasx_xvmod_h asm-fmts = xd, xj, xk data-types = V16HI, V16HI, V16HI /// lasx_xvmod_w -impl = portable name = lasx_xvmod_w asm-fmts = xd, xj, xk data-types = V8SI, V8SI, V8SI /// lasx_xvmod_d -impl = portable name = lasx_xvmod_d asm-fmts = xd, xj, xk data-types = V4DI, V4DI, V4DI /// lasx_xvmod_bu -impl = portable name = lasx_xvmod_bu asm-fmts = xd, xj, xk data-types = UV32QI, UV32QI, UV32QI /// lasx_xvmod_hu -impl = portable name = lasx_xvmod_hu asm-fmts = xd, xj, xk data-types = UV16HI, UV16HI, UV16HI /// lasx_xvmod_wu -impl = portable name = lasx_xvmod_wu asm-fmts = xd, xj, xk data-types = UV8SI, UV8SI, UV8SI /// lasx_xvmod_du -impl = portable name = lasx_xvmod_du asm-fmts = xd, xj, xk data-types = UV4DI, UV4DI, UV4DI diff --git a/library/stdarch/crates/stdarch-gen-loongarch/lsx.spec b/library/stdarch/crates/stdarch-gen-loongarch/lsx.spec index 89970c3657048..e1fb72c7901c9 100644 --- a/library/stdarch/crates/stdarch-gen-loongarch/lsx.spec +++ b/library/stdarch/crates/stdarch-gen-loongarch/lsx.spec @@ -1260,49 +1260,41 @@ asm-fmts = vd, vj, vk data-types = V2DI, V2DI, V2DI, V2DI /// lsx_vdiv_b -impl = portable name = lsx_vdiv_b asm-fmts = vd, vj, vk data-types = V16QI, V16QI, V16QI /// lsx_vdiv_h -impl = portable name = lsx_vdiv_h asm-fmts = vd, vj, vk data-types = V8HI, V8HI, V8HI /// lsx_vdiv_w -impl = portable name = lsx_vdiv_w asm-fmts = vd, vj, vk data-types = V4SI, V4SI, V4SI /// lsx_vdiv_d -impl = portable name = lsx_vdiv_d asm-fmts = vd, vj, vk data-types = V2DI, V2DI, V2DI /// lsx_vdiv_bu -impl = portable name = lsx_vdiv_bu asm-fmts = vd, vj, vk data-types = UV16QI, UV16QI, UV16QI /// lsx_vdiv_hu -impl = portable name = lsx_vdiv_hu asm-fmts = vd, vj, vk data-types = UV8HI, UV8HI, UV8HI /// lsx_vdiv_wu -impl = portable name = lsx_vdiv_wu asm-fmts = vd, vj, vk data-types = UV4SI, UV4SI, UV4SI /// lsx_vdiv_du -impl = portable name = lsx_vdiv_du asm-fmts = vd, vj, vk data-types = UV2DI, UV2DI, UV2DI @@ -1368,49 +1360,41 @@ asm-fmts = vd, vj, vk data-types = V2DI, UV4SI, UV4SI /// lsx_vmod_b -impl = portable name = lsx_vmod_b asm-fmts = vd, vj, vk data-types = V16QI, V16QI, V16QI /// lsx_vmod_h -impl = portable name = lsx_vmod_h asm-fmts = vd, vj, vk data-types = V8HI, V8HI, V8HI /// lsx_vmod_w -impl = portable name = lsx_vmod_w asm-fmts = vd, vj, vk data-types = V4SI, V4SI, V4SI /// lsx_vmod_d -impl = portable name = lsx_vmod_d asm-fmts = vd, vj, vk data-types = V2DI, V2DI, V2DI /// lsx_vmod_bu -impl = portable name = lsx_vmod_bu asm-fmts = vd, vj, vk data-types = UV16QI, UV16QI, UV16QI /// lsx_vmod_hu -impl = portable name = lsx_vmod_hu asm-fmts = vd, vj, vk data-types = UV8HI, UV8HI, UV8HI /// lsx_vmod_wu -impl = portable name = lsx_vmod_wu asm-fmts = vd, vj, vk data-types = UV4SI, UV4SI, UV4SI /// lsx_vmod_du -impl = portable name = lsx_vmod_du asm-fmts = vd, vj, vk data-types = UV2DI, UV2DI, UV2DI diff --git a/library/stdarch/crates/stdarch-gen-loongarch/src/portable-intrinsics.txt b/library/stdarch/crates/stdarch-gen-loongarch/src/portable-intrinsics.txt index f47c17b23adea..4005385b75198 100644 --- a/library/stdarch/crates/stdarch-gen-loongarch/src/portable-intrinsics.txt +++ b/library/stdarch/crates/stdarch-gen-loongarch/src/portable-intrinsics.txt @@ -111,22 +111,6 @@ lsx_vmul_b lsx_vmul_h lsx_vmul_w lsx_vmul_d -lsx_vdiv_b -lsx_vdiv_h -lsx_vdiv_w -lsx_vdiv_d -lsx_vdiv_bu -lsx_vdiv_hu -lsx_vdiv_wu -lsx_vdiv_du -lsx_vmod_b -lsx_vmod_h -lsx_vmod_w -lsx_vmod_d -lsx_vmod_bu -lsx_vmod_hu -lsx_vmod_wu -lsx_vmod_du lsx_vmadd_b lsx_vmadd_h lsx_vmadd_w @@ -426,22 +410,6 @@ lasx_xvmul_b lasx_xvmul_h lasx_xvmul_w lasx_xvmul_d -lasx_xvdiv_b -lasx_xvdiv_h -lasx_xvdiv_w -lasx_xvdiv_d -lasx_xvdiv_bu -lasx_xvdiv_hu -lasx_xvdiv_wu -lasx_xvdiv_du -lasx_xvmod_b -lasx_xvmod_h -lasx_xvmod_w -lasx_xvmod_d -lasx_xvmod_bu -lasx_xvmod_hu -lasx_xvmod_wu -lasx_xvmod_du lasx_xvmadd_b lasx_xvmadd_h lasx_xvmadd_w diff --git a/library/stdarch/rust-version b/library/stdarch/rust-version index 368cecc0b82f9..271b3a644cfb8 100644 --- a/library/stdarch/rust-version +++ b/library/stdarch/rust-version @@ -1 +1 @@ -220b36c420c49c59923f54cd4a76634fac98a067 +260f1acad9d7f70b5b56a636a9dc5a7f76150417 diff --git a/library/stdarch/triagebot.toml b/library/stdarch/triagebot.toml index 149646aa65f01..c4a4b02b0179e 100644 --- a/library/stdarch/triagebot.toml +++ b/library/stdarch/triagebot.toml @@ -33,7 +33,7 @@ allow-unauthenticated = [ [assign] [assign.owners] -"*" = ["@Amanieu", "@folkertdev", "@sayantn", "@davidtwco", "@adamgemmell"] +"*" = ["@Amanieu", "@folkertdev", "@davidtwco", "@adamgemmell"] [ping.windows] message = """\ @@ -88,3 +88,13 @@ Thanks! # Documentation at: https://forge.rust-lang.org/triagebot/issue-links.html [issue-links] check-commits = false + +[notify-zulip."I-libs-nominated"] +zulip_stream = 639962 # #t-libs/nominated +topic = "stdarch#{number}: {title}" +message_on_add = """\ +[stdarch#{number} "{title}"](https://github.com/rust-lang/stdarch/pull/{number}) has been nominated for discussion. +""" +message_on_remove = """\ +stdarch#{number} is no longer nominated. +""" diff --git a/library/test/src/lib.rs b/library/test/src/lib.rs index 7eb632ba69882..eea136c450121 100644 --- a/library/test/src/lib.rs +++ b/library/test/src/lib.rs @@ -616,10 +616,11 @@ pub fn run_test( ) -> Option> { let TestDescAndFn { desc, testfn } = test; - // Emscripten can catch panics but other wasm targets cannot + // Emscripten can catch panics, and test for `panic=unwind` on other wasm + // targets to see if they can catch panics. let ignore_because_no_process_support = desc.should_panic != ShouldPanic::No && (cfg!(target_family = "wasm") || cfg!(target_os = "zkvm")) - && !cfg!(target_os = "emscripten"); + && !(cfg!(target_os = "emscripten") || cfg!(panic = "unwind")); if force_ignore || desc.ignore || ignore_because_no_process_support { let message = CompletedTest::new(id, desc, TrIgnored, None, Vec::new()); @@ -667,7 +668,8 @@ pub fn run_test( // level. let supports_threads = !cfg!(target_os = "emscripten") && !cfg!(target_family = "wasm") - && !cfg!(target_os = "zkvm"); + && !cfg!(target_os = "zkvm") + && !cfg!(target_os = "uefi"); if supports_threads { let cfg = thread::Builder::new().name(name.as_slice().to_owned()); let mut runtest = Arc::new(Mutex::new(Some(runtest))); diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 8013d9cc08638..a915053a5222b 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -2,5 +2,5 @@ # standard library we currently track. [toolchain] -channel = "nightly-2026-10-12" +channel = "nightly-2026-10-22" components = ["llvm-tools-preview", "rustc-dev", "rust-src", "rustfmt"] diff --git a/tool_config/kani-version.toml b/tool_config/kani-version.toml index e121de6e60e9a..0ebc05137bda6 100644 --- a/tool_config/kani-version.toml +++ b/tool_config/kani-version.toml @@ -2,4 +2,4 @@ # incompatible with the verify-std repo. [kani] -commit = "9461e07af7ec929f050aa4192f29363c9be07779" +commit = "cd355b0bccd32300713bc2ce4f765dafa1510607"