diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb6bc325a3bb7..eb2170f8ec8e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -154,6 +154,11 @@ jobs: - name: ensure the channel matches the target branch run: src/ci/scripts/verify-channel.sh + - name: print kernel/libc versions + if: runner.os == 'Linux' + run: | + uname -a + - name: collect CPU statistics run: src/ci/scripts/collect-cpu-stats.sh diff --git a/library/std/src/fs/tests.rs b/library/std/src/fs/tests.rs index 0a5d1153d860c..26bc2bf9c013b 100644 --- a/library/std/src/fs/tests.rs +++ b/library/std/src/fs/tests.rs @@ -1,6 +1,7 @@ use rand::RngCore; use crate::assert_matches::assert_matches; +use crate::bstr::ByteStr; use crate::fs::{self, File, FileTimes, OpenOptions, TryLockError}; use crate::io::prelude::*; use crate::io::{BorrowedBuf, ErrorKind, SeekFrom}; @@ -589,6 +590,25 @@ fn test_read_buf_exact_at() { check!(fs::remove_file(&filename)); } +#[test] +#[cfg(unix)] +fn file_test_append_write_at() { + use crate::os::unix::fs::FileExt; + + let tmpdir = tmpdir(); + let filename = tmpdir.join("file_test_append_write_at.txt"); + let msg = b"it's not working!"; + check!(fs::write(&filename, &msg)); + // write_at should work even in in append mode + let mut f = check!(fs::File::options().append(true).open(&filename)); + assert_eq!(check!(f.stream_position()), 0); + assert_eq!(check!(f.write_at(b" ", 5)), 3); + assert_eq!(check!(f.stream_position()), 0); + + let content = check!(fs::read(&filename)); + assert_eq!(ByteStr::new(&content), ByteStr::new(b"it's working!")); +} + #[test] #[cfg(unix)] fn set_get_unix_permissions() { diff --git a/library/std/src/os/unix/fs.rs b/library/std/src/os/unix/fs.rs index 219b340b92469..d97905b21b9ee 100644 --- a/library/std/src/os/unix/fs.rs +++ b/library/std/src/os/unix/fs.rs @@ -233,9 +233,10 @@ pub trait FileExt { /// /// # Bug /// On some systems, `write_at` utilises [`pwrite64`] to write to files. - /// However, this syscall has a [bug] where files opened with the `O_APPEND` + /// However, on linux this syscall has a [bug] where files opened with the `O_APPEND` /// flag fail to respect the offset parameter, always appending to the end - /// of the file instead. + /// of the file instead. When available `pwritev2(..., RWF_NOAPPEND)` will + /// be used instead to avoid this bug. /// /// It is possible to inadvertently set this flag, like in the example below. /// Therefore, it is important to be vigilant while changing options to mitigate diff --git a/library/std/src/os/unix/fs/tests.rs b/library/std/src/os/unix/fs/tests.rs index 1840bb38c17c8..ecd799049282a 100644 --- a/library/std/src/os/unix/fs/tests.rs +++ b/library/std/src/os/unix/fs/tests.rs @@ -1,4 +1,6 @@ use super::*; +use crate::bstr::ByteStr; +use crate::io::Seek; #[test] fn read_vectored_at() { @@ -39,13 +41,16 @@ fn write_vectored_at() { file.write_all(msg).unwrap(); } let expected = { - let file = fs::File::options().write(true).open(&filename).unwrap(); + // Open in append mode to test that positioned writes bypass O_APPEND. + let mut file = fs::File::options().append(true).open(&filename).unwrap(); let buf0 = b" "; let buf1 = b"great "; let iovec = [io::IoSlice::new(buf0), io::IoSlice::new(buf1)]; + assert_eq!(file.stream_position().unwrap(), 0); let n = file.write_vectored_at(&iovec, 11).unwrap(); + assert_eq!(file.stream_position().unwrap(), 0); assert!(n == 4 || n == 11); @@ -53,7 +58,7 @@ fn write_vectored_at() { }; let content = fs::read(&filename).unwrap(); - assert_eq!(&content, expected); + assert_eq!(ByteStr::new(&content), ByteStr::new(expected)); } #[test] diff --git a/library/std/src/sys/fd/unix.rs b/library/std/src/sys/fd/unix.rs index bb6c0ac9e18e6..bc2b0a71b5616 100644 --- a/library/std/src/sys/fd/unix.rs +++ b/library/std/src/sys/fd/unix.rs @@ -36,9 +36,12 @@ cfg_select! { use crate::cmp; use crate::io::{self, BorrowedCursor, IoSlice, IoSliceMut, Read}; use crate::os::fd::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; -#[cfg(all(target_os = "android", target_pointer_width = "64"))] +#[cfg(all(any(target_os = "android", target_os = "linux"), target_pointer_width = "64"))] use crate::sys::pal::weak::syscall; -#[cfg(any(all(target_os = "android", target_pointer_width = "32"), target_vendor = "apple"))] +#[cfg(any( + all(any(target_os = "android", target_os = "linux"), target_pointer_width = "32"), + target_vendor = "apple" +))] use crate::sys::pal::weak::weak; use crate::sys::{AsInner, FromInner, IntoInner, cvt}; @@ -404,6 +407,16 @@ impl FileDesc { ))] use libc::pwrite64; + // Work around linux deviating from POSIX where it ignores the + // offset of pwrite when the file was opened with O_APPEND. + #[cfg(any(target_os = "linux", target_os = "android"))] + { + let iov = [IoSlice::new(buf)]; + if let Some(ret) = self.pwritev2(&iov, offset) { + return ret; + } + } + unsafe { cvt(pwrite64( self.as_raw_fd(), @@ -428,6 +441,13 @@ impl FileDesc { target_os = "openbsd", // OpenBSD 2.7 ))] pub fn write_vectored_at(&self, bufs: &[IoSlice<'_>], offset: u64) -> io::Result { + // Work around linux deviating from POSIX where it ignores the + // offset of pwrite when the file was opened with O_APPEND. + #[cfg(any(target_os = "linux", target_os = "android"))] + if let Some(ret) = self.pwritev2(bufs, offset) { + return ret; + } + let ret = cvt(unsafe { libc::pwritev( self.as_raw_fd(), @@ -633,6 +653,65 @@ impl FileDesc { pub fn duplicate(&self) -> io::Result { Ok(Self(self.0.try_clone()?)) } + + #[cfg(any(target_os = "linux", target_os = "android"))] + fn pwritev2(&self, bufs: &[IoSlice<'_>], offset: u64) -> Option> { + #[cfg(target_pointer_width = "64")] + syscall!( + fn pwritev2( + fd: libc::c_int, + iovec: *const libc::iovec, + n_iovec: libc::c_int, + offset: off64_t, + flags: libc::c_int, + ) -> isize; + ); + #[cfg(target_pointer_width = "32")] + let pwritev2 = { + weak!( + fn pwritev2( + fd: libc::c_int, + iovec: *const libc::iovec, + n_iovec: libc::c_int, + offset: off64_t, + flags: libc::c_int, + ) -> isize; + ); + let Some(pwritev2) = pwritev2.get() else { + return None; + }; + pwritev2 + }; + + use core::sync::atomic::AtomicBool; + + static NOAPPEND_SUPPORTED: AtomicBool = AtomicBool::new(true); + if NOAPPEND_SUPPORTED.load(core::sync::atomic::Ordering::Relaxed) { + let r = unsafe { + cvt(pwritev2( + self.as_raw_fd(), + bufs.as_ptr() as *const libc::iovec, + cmp::min(bufs.len(), max_iov()) as libc::c_int, + offset as off64_t, + libc::RWF_NOAPPEND, + )) + }; + match r { + Ok(ret) => return Some(Ok(ret as usize)), + Err(e) + if let Some(err) = e.raw_os_error() + && (err == libc::EOPNOTSUPP || err == libc::ENOSYS) => + { + eprintln!("pwritev2 NOAPPEND error: {err}"); + NOAPPEND_SUPPORTED.store(false, core::sync::atomic::Ordering::Relaxed); + return None; + } + Err(e) => return Some(Err(e)), + } + } + + return None; + } } impl<'a> Read for &'a FileDesc { diff --git a/src/ci/docker/host-x86_64/pr-check-2/Dockerfile b/src/ci/docker/host-x86_64/pr-check-2/Dockerfile index 68162d136c3f4..2d4f9ca9c0020 100644 --- a/src/ci/docker/host-x86_64/pr-check-2/Dockerfile +++ b/src/ci/docker/host-x86_64/pr-check-2/Dockerfile @@ -18,6 +18,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ xz-utils \ libssl-dev \ pkg-config \ + strace \ mingw-w64 \ && rm -rf /var/lib/apt/lists/* @@ -31,21 +32,6 @@ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh ENV SCRIPT \ - python3 ../x.py check && \ - python3 ../x.py clippy ci --stage 2 && \ - python3 ../x.py test --stage 1 core alloc std test proc_macro && \ - # Elsewhere, we run all tests for the host. A number of codegen tests are sensitive to the target pointer - # width, for example because they mention a usize. wasm32-wasip1 in test-various, so using it here can't make - # PR CI more strict than full CI. - python3 ../x.py test --stage 1 tests/codegen-llvm --target wasm32-wasip1 && \ - python3 ../x.py test --stage 1 src/tools/compiletest && \ - python3 ../x.py doc bootstrap --stage 1 && \ - # Build both public and internal documentation. - RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc compiler --stage 1 && \ - RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc library --stage 1 && \ - mkdir -p /checkout/obj/staging/doc && \ - cp -r build/x86_64-unknown-linux-gnu/doc /checkout/obj/staging && \ - RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc library/test --stage 1 && \ - # The BOOTSTRAP_TRACING flag is added to verify whether the - # bootstrap process compiles successfully with this flag enabled. - BOOTSTRAP_TRACING=1 python3 ../x.py --help + ldd --version && \ + python3 ../x.py test --stage 1 std -- "write_at" || \ + strace -ffe write,pwritev2,pwrite64,pwritev python3 ../x.py test --stage 1 std -- "write_at"