From f9f51208d5260d5bb2ed7389534680f17cbc005e Mon Sep 17 00:00:00 2001 From: Daringcuteseal Date: Fri, 20 Dec 2024 08:27:49 +0700 Subject: [PATCH 1/7] uucore/buf_copy: add platform-specific compile guards --- src/uucore/src/lib/features.rs | 4 +- src/uucore/src/lib/features/buf_copy.rs | 130 +++++++++++++++++++----- src/uucore/src/lib/lib.rs | 4 +- 3 files changed, 110 insertions(+), 28 deletions(-) diff --git a/src/uucore/src/lib/features.rs b/src/uucore/src/lib/features.rs index cde1cf264a3..ef5be724d9f 100644 --- a/src/uucore/src/lib/features.rs +++ b/src/uucore/src/lib/features.rs @@ -6,6 +6,8 @@ #[cfg(feature = "backup-control")] pub mod backup_control; +#[cfg(feature = "buf-copy")] +pub mod buf_copy; #[cfg(feature = "checksum")] pub mod checksum; #[cfg(feature = "colors")] @@ -39,8 +41,6 @@ pub mod version_cmp; pub mod mode; // ** unix-only -#[cfg(all(any(target_os = "linux", target_os = "android"), feature = "buf-copy"))] -pub mod buf_copy; #[cfg(all(unix, feature = "entries"))] pub mod entries; #[cfg(all(unix, feature = "perms"))] diff --git a/src/uucore/src/lib/features/buf_copy.rs b/src/uucore/src/lib/features/buf_copy.rs index 2b46248a597..827626a21b1 100644 --- a/src/uucore/src/lib/features/buf_copy.rs +++ b/src/uucore/src/lib/features/buf_copy.rs @@ -9,27 +9,55 @@ //! used by utilities to work around the limitations of Rust's `fs::copy` which //! does not handle copying special files (e.g pipes, character/block devices). -use crate::error::{UError, UResult}; -use nix::unistd; +use crate::error::UError; +use std::io::{Read, Write}; + +#[cfg(any(target_os = "linux", target_os = "android"))] +use super::pipes::{pipe, splice, splice_exact, vmsplice}; +#[cfg(any(target_os = "linux", target_os = "android"))] +use nix::errno::Errno; +#[cfg(any(target_os = "linux", target_os = "android"))] +use nix::{libc::S_IFIFO, sys::stat::fstat, unistd}; +#[cfg(any(target_os = "linux", target_os = "android"))] use std::fs::File; -use std::{ - io::{self, Read, Write}, - os::{ - fd::AsFd, - unix::io::{AsRawFd, RawFd}, - }, -}; +#[cfg(any(target_os = "linux", target_os = "android"))] +use std::os::unix::io::RawFd; +#[cfg(any(target_os = "linux", target_os = "android"))] +type Result = std::result::Result; -use nix::{errno::Errno, libc::S_IFIFO, sys::stat::fstat}; +use crate::error::UResult; -use super::pipes::{pipe, splice, splice_exact, vmsplice}; +#[cfg(unix)] +use std::os::unix::io::{AsFd, AsRawFd}; -type Result = std::result::Result; +#[cfg(unix)] +/// A readable file descriptor. Available in unix and unix-line platforms. +pub trait FdReadable: Read + AsFd + AsRawFd {} +#[cfg(not(unix))] +/// A readable file descriptor. Available in non-unix platforms. +pub trait FdReadable: Read {} + +#[cfg(unix)] +impl FdReadable for T where T: Read + AsFd + AsRawFd {} +#[cfg(not(unix))] +impl FdReadable for T where T: Read {} + +#[cfg(unix)] +/// A writable file descriptor. Available in unix and unix-line platforms. +pub trait FdWritable: Write + AsFd + AsRawFd {} +#[cfg(not(unix))] +/// A writable file descriptor. Available in non-unix platforms. +pub trait FdWritable: Write {} + +#[cfg(unix)] +impl FdWritable for T where T: Write + AsFd + AsRawFd {} +#[cfg(not(unix))] +impl FdWritable for T where T: Write {} /// Error types used by buffer-copying functions from the `buf_copy` module. #[derive(Debug)] pub enum Error { - Io(io::Error), + Io(std::io::Error), WriteError(String), } @@ -54,7 +82,17 @@ impl UError for Error { } } -/// Helper function to determine whether a given handle (such as a file) is a pipe or not. +// The generalization of this function (and other splice_data functions) is not trivial as most +// utilities will just write data finitely. However, `yes`, which is the sole crate using these +// functions as of now, continuously loops the data write. Coupling the `is_pipe` check together +// with the data write logic means that the check has to be done for every single write, which adds +// unnecessary overhead. +// +/// Helper function to determine whether a given handle (such as a file) is a pipe or not. Can be +/// used to determine whether to use the `splice_data_to_pipe` or the `splice_data_to_fd` function. +/// This function is available exclusively to Linux and Android as it is meant to be used at the +/// scope of splice operations. +/// /// /// # Arguments /// * `out` - path of handle @@ -62,7 +100,7 @@ impl UError for Error { /// # Returns /// A `bool` indicating whether the given handle is a pipe or not. #[inline] -#[cfg(unix)] +#[cfg(any(target_os = "linux", target_os = "android"))] pub fn is_pipe

(path: &P) -> Result where P: AsRawFd, @@ -70,7 +108,9 @@ where Ok(fstat(path.as_raw_fd())?.st_mode as nix::libc::mode_t & S_IFIFO != 0) } +#[cfg(any(target_os = "linux", target_os = "android"))] const SPLICE_SIZE: usize = 1024 * 128; +#[cfg(any(target_os = "linux", target_os = "android"))] const BUF_SIZE: usize = 1024 * 16; /// Copy data from `Read` implementor `source` into a `Write` implementor @@ -92,8 +132,8 @@ const BUF_SIZE: usize = 1024 * 16; /// operation is successful. pub fn copy_stream(src: &mut R, dest: &mut S) -> UResult where - R: Read + AsFd + AsRawFd, - S: Write + AsFd + AsRawFd, + R: FdReadable, + S: FdWritable, { #[cfg(any(target_os = "linux", target_os = "android"))] { @@ -160,6 +200,7 @@ where /// Move exactly `num_bytes` bytes from `read_fd` to `write_fd` using the `read` /// and `write` calls. +#[cfg(any(target_os = "linux", target_os = "android"))] fn copy_exact(read_fd: RawFd, write_fd: &impl AsFd, num_bytes: usize) -> std::io::Result { let mut left = num_bytes; let mut buf = [0; BUF_SIZE]; @@ -247,10 +288,10 @@ pub fn splice_data_to_fd( } /// Conversion from a `nix::Error` into our `Error` which implements `UError`. -#[cfg(any(target_os = "linux", target_os = "android"))] +#[cfg(unix)] impl From for Error { fn from(error: nix::Error) -> Self { - Self::Io(io::Error::from_raw_os_error(error as i32)) + Self::Io(std::io::Error::from_raw_os_error(error as i32)) } } @@ -276,16 +317,22 @@ fn maybe_unsupported(error: nix::Error) -> Result<(u64, bool)> { #[cfg(test)] mod tests { + use super::*; + #[cfg(any(target_os = "linux", target_os = "android", not(unix)))] + use std::fs::File; + #[cfg(any(target_os = "linux", target_os = "android", not(unix)))] use tempfile::tempdir; - use super::*; + #[cfg(unix)] use crate::pipes; + #[cfg(any(target_os = "linux", target_os = "android"))] fn new_temp_file() -> File { let temp_dir = tempdir().unwrap(); File::create(temp_dir.path().join("file.txt")).unwrap() } + #[cfg(any(target_os = "linux", target_os = "android"))] #[test] fn test_file_is_pipe() { let temp_file = new_temp_file(); @@ -296,21 +343,26 @@ mod tests { assert!(!is_pipe(&temp_file).unwrap()); } + #[cfg(any(target_os = "linux", target_os = "android"))] #[test] fn test_valid_splice_errs() { - let err = nix::Error::from(Errno::EINVAL); + use nix::errno::Errno; + use nix::Error; + + let err = Error::from(Errno::EINVAL); assert_eq!(maybe_unsupported(err).unwrap(), (0, true)); - let err = nix::Error::from(Errno::ENOSYS); + let err = Error::from(Errno::ENOSYS); assert_eq!(maybe_unsupported(err).unwrap(), (0, true)); - let err = nix::Error::from(Errno::EBADF); + let err = Error::from(Errno::EBADF); assert_eq!(maybe_unsupported(err).unwrap(), (0, true)); - let err = nix::Error::from(Errno::EPERM); + let err = Error::from(Errno::EPERM); assert!(maybe_unsupported(err).is_err()); } + #[cfg(any(target_os = "linux", target_os = "android"))] #[test] fn test_splice_data_to_pipe() { let (pipe_read, pipe_write) = pipes::pipe().unwrap(); @@ -322,6 +374,7 @@ mod tests { assert_eq!(bytes as usize, data.len()); } + #[cfg(any(target_os = "linux", target_os = "android"))] #[test] fn test_splice_data_to_file() { let mut temp_file = new_temp_file(); @@ -334,6 +387,7 @@ mod tests { assert_eq!(bytes as usize, data.len()); } + #[cfg(any(target_os = "linux", target_os = "android"))] #[test] fn test_copy_exact() { let (mut pipe_read, mut pipe_write) = pipes::pipe().unwrap(); @@ -348,6 +402,7 @@ mod tests { } #[test] + #[cfg(unix)] fn test_copy_stream() { let (mut pipe_read, mut pipe_write) = pipes::pipe().unwrap(); let data = b"Hello, world!"; @@ -360,6 +415,33 @@ mod tests { assert_eq!(&buf[..n as usize], data); } + #[test] + #[cfg(not(unix))] + // Test for non-unix platforms. We use regular files instead. + fn test_copy_stream() { + let temp_dir = tempdir().unwrap(); + let src_path = temp_dir.path().join("src.txt"); + let dest_path = temp_dir.path().join("dest.txt"); + + let mut src_file = File::create(&src_path).unwrap(); + let mut dest_file = File::create(&dest_path).unwrap(); + + let data = b"Hello, world!"; + src_file.write_all(data).unwrap(); + src_file.sync_all().unwrap(); + + let mut src_file = File::open(&src_path).unwrap(); + let bytes_copied = copy_stream(&mut src_file, &mut dest_file).unwrap(); + + let mut dest_file = File::open(&dest_path).unwrap(); + let mut buf = Vec::new(); + dest_file.read_to_end(&mut buf).unwrap(); + + assert_eq!(bytes_copied as usize, data.len()); + assert_eq!(buf, data); + } + + #[cfg(any(target_os = "linux", target_os = "android"))] #[test] fn test_splice_write() { let (mut pipe_read, pipe_write) = pipes::pipe().unwrap(); diff --git a/src/uucore/src/lib/lib.rs b/src/uucore/src/lib/lib.rs index 3a6a537ad49..684de8f74e0 100644 --- a/src/uucore/src/lib/lib.rs +++ b/src/uucore/src/lib/lib.rs @@ -40,6 +40,8 @@ pub use crate::parser::shortcut_value_parser; // * feature-gated modules #[cfg(feature = "backup-control")] pub use crate::features::backup_control; +#[cfg(feature = "buf-copy")] +pub use crate::features::buf_copy; #[cfg(feature = "checksum")] pub use crate::features::checksum; #[cfg(feature = "colors")] @@ -70,8 +72,6 @@ pub use crate::features::version_cmp; #[cfg(all(not(windows), feature = "mode"))] pub use crate::features::mode; // ** unix-only -#[cfg(all(any(target_os = "linux", target_os = "android"), feature = "buf-copy"))] -pub use crate::features::buf_copy; #[cfg(all(unix, feature = "entries"))] pub use crate::features::entries; #[cfg(all(unix, feature = "perms"))] From d5ca1d70dab84ed041d72cbc1473a5ca296bd21d Mon Sep 17 00:00:00 2001 From: Daringcuteseal Date: Fri, 20 Dec 2024 12:01:30 +0700 Subject: [PATCH 2/7] uucore/buf_copy: fix unnecessary loop statement the splice_data_to_fd function now does not loop forever and returns the amount of bytes written properly. --- src/uucore/src/lib/features/buf_copy.rs | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/uucore/src/lib/features/buf_copy.rs b/src/uucore/src/lib/features/buf_copy.rs index 827626a21b1..50152d1bf69 100644 --- a/src/uucore/src/lib/features/buf_copy.rs +++ b/src/uucore/src/lib/features/buf_copy.rs @@ -272,19 +272,20 @@ pub fn splice_data_to_fd( write_pipe: &File, dest: &T, ) -> UResult<(u64, bool)> { - loop { - let mut bytes = bytes; - while !bytes.is_empty() { - let len = match vmsplice(&write_pipe, bytes) { - Ok(n) => n, - Err(e) => return Ok(maybe_unsupported(e)?), - }; - if let Err(e) = splice_exact(&read_pipe, dest, len) { - return Ok(maybe_unsupported(e)?); - } - bytes = &bytes[len..]; + let mut n_bytes: u64 = 0; + let mut bytes = bytes; + while !bytes.is_empty() { + let len = match vmsplice(&write_pipe, bytes) { + Ok(n) => n, + Err(e) => return Ok(maybe_unsupported(e)?), + }; + if let Err(e) = splice_exact(&read_pipe, dest, len) { + return Ok(maybe_unsupported(e)?); } + bytes = &bytes[len..]; + n_bytes += len as u64; } + Ok((n_bytes, false)) } /// Conversion from a `nix::Error` into our `Error` which implements `UError`. From 15031e01b63308b28eadf8f4b5dc3c3522d535bc Mon Sep 17 00:00:00 2001 From: Daringcuteseal Date: Fri, 20 Dec 2024 12:03:53 +0700 Subject: [PATCH 3/7] uucore/buf_copy: fix hanging tests several tests now are executed properly with proper handling of pipelines. --- src/uucore/src/lib/features/buf_copy.rs | 80 ++++++++++++++++++------- 1 file changed, 60 insertions(+), 20 deletions(-) diff --git a/src/uucore/src/lib/features/buf_copy.rs b/src/uucore/src/lib/features/buf_copy.rs index 50152d1bf69..5b18733f4d0 100644 --- a/src/uucore/src/lib/features/buf_copy.rs +++ b/src/uucore/src/lib/features/buf_copy.rs @@ -319,18 +319,23 @@ fn maybe_unsupported(error: nix::Error) -> Result<(u64, bool)> { #[cfg(test)] mod tests { use super::*; - #[cfg(any(target_os = "linux", target_os = "android", not(unix)))] use std::fs::File; - #[cfg(any(target_os = "linux", target_os = "android", not(unix)))] use tempfile::tempdir; #[cfg(unix)] use crate::pipes; + #[cfg(unix)] + use std::fs::OpenOptions; - #[cfg(any(target_os = "linux", target_os = "android"))] + #[cfg(unix)] fn new_temp_file() -> File { let temp_dir = tempdir().unwrap(); - File::create(temp_dir.path().join("file.txt")).unwrap() + OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(temp_dir.path().join("file.txt")) + .unwrap() } #[cfg(any(target_os = "linux", target_os = "android"))] @@ -378,14 +383,20 @@ mod tests { #[cfg(any(target_os = "linux", target_os = "android"))] #[test] fn test_splice_data_to_file() { + use std::io::{Seek, SeekFrom}; + let mut temp_file = new_temp_file(); let (pipe_read, pipe_write) = pipes::pipe().unwrap(); let data = b"Hello, world!"; let (bytes, _) = splice_data_to_fd(data, &pipe_read, &pipe_write, &temp_file).unwrap(); - let mut buf = [0; 1024]; - let n = temp_file.read(&mut buf).unwrap(); - assert_eq!(&buf[..n], data); assert_eq!(bytes as usize, data.len()); + + // We would have been at the end already, so seek again to the start. + temp_file.seek(SeekFrom::Start(0)).unwrap(); + + let mut buf = Vec::new(); + temp_file.read_to_end(&mut buf).unwrap(); + assert_eq!(buf, data); } #[cfg(any(target_os = "linux", target_os = "android"))] @@ -405,15 +416,28 @@ mod tests { #[test] #[cfg(unix)] fn test_copy_stream() { + use std::{ + io::{Seek, SeekFrom}, + thread, + }; + let mut dest_file = new_temp_file(); + let (mut pipe_read, mut pipe_write) = pipes::pipe().unwrap(); let data = b"Hello, world!"; - let n = pipe_write.write(data).unwrap(); - assert_eq!(n, data.len()); - let mut buf = [0; 1024]; - let n = copy_stream(&mut pipe_read, &mut pipe_write).unwrap(); - let n2 = pipe_read.read(&mut buf).unwrap(); - assert_eq!(n as usize, n2); - assert_eq!(&buf[..n as usize], data); + let thread = thread::spawn(move || { + pipe_write.write_all(data).unwrap(); + }); + let result = copy_stream(&mut pipe_read, &mut dest_file).unwrap(); + thread.join().unwrap(); + assert!(result == data.len() as u64); + + // We would have been at the end already, so seek again to the start. + dest_file.seek(SeekFrom::Start(0)).unwrap(); + + let mut buf = Vec::new(); + dest_file.read_to_end(&mut buf).unwrap(); + + assert_eq!(buf, data); } #[test] @@ -445,12 +469,28 @@ mod tests { #[cfg(any(target_os = "linux", target_os = "android"))] #[test] fn test_splice_write() { - let (mut pipe_read, pipe_write) = pipes::pipe().unwrap(); + use std::{ + io::{Seek, SeekFrom}, + thread, + }; + + let (pipe_read, mut pipe_write) = pipes::pipe().unwrap(); + let mut dest_file = new_temp_file(); let data = b"Hello, world!"; - let (bytes, _) = splice_write(&pipe_read, &pipe_write).unwrap(); - let mut buf = [0; 1024]; - let n = pipe_read.read(&mut buf).unwrap(); - assert_eq!(&buf[..n], data); - assert_eq!(bytes as usize, data.len()); + let thread = thread::spawn(move || { + pipe_write.write_all(data).unwrap(); + }); + let (bytes, _) = splice_write(&pipe_read, &dest_file).unwrap(); + thread.join().unwrap(); + + assert!(bytes == data.len() as u64); + + // We would have been at the end already, so seek again to the start. + dest_file.seek(SeekFrom::Start(0)).unwrap(); + + let mut buf = Vec::new(); + dest_file.read_to_end(&mut buf).unwrap(); + + assert_eq!(buf, data); } } From 922341e2e4f6b65ee0f0b1cafca63baa9429b179 Mon Sep 17 00:00:00 2001 From: Daringcuteseal Date: Mon, 23 Dec 2024 07:48:03 +0700 Subject: [PATCH 4/7] uucore/buf_copy: split into platform-specific files refactor the buf_copy module into Linux and non-Linux implementations. --- src/uucore/src/lib/features/buf_copy.rs | 488 +----------------- .../src/lib/features/buf_copy/common.rs | 29 ++ src/uucore/src/lib/features/buf_copy/linux.rs | 255 +++++++++ src/uucore/src/lib/features/buf_copy/other.rs | 25 + 4 files changed, 316 insertions(+), 481 deletions(-) create mode 100644 src/uucore/src/lib/features/buf_copy/common.rs create mode 100644 src/uucore/src/lib/features/buf_copy/linux.rs create mode 100644 src/uucore/src/lib/features/buf_copy/other.rs diff --git a/src/uucore/src/lib/features/buf_copy.rs b/src/uucore/src/lib/features/buf_copy.rs index 5b18733f4d0..5ea5d8eaa12 100644 --- a/src/uucore/src/lib/features/buf_copy.rs +++ b/src/uucore/src/lib/features/buf_copy.rs @@ -9,488 +9,14 @@ //! used by utilities to work around the limitations of Rust's `fs::copy` which //! does not handle copying special files (e.g pipes, character/block devices). -use crate::error::UError; -use std::io::{Read, Write}; +pub mod common; #[cfg(any(target_os = "linux", target_os = "android"))] -use super::pipes::{pipe, splice, splice_exact, vmsplice}; +pub mod linux; #[cfg(any(target_os = "linux", target_os = "android"))] -use nix::errno::Errno; -#[cfg(any(target_os = "linux", target_os = "android"))] -use nix::{libc::S_IFIFO, sys::stat::fstat, unistd}; -#[cfg(any(target_os = "linux", target_os = "android"))] -use std::fs::File; -#[cfg(any(target_os = "linux", target_os = "android"))] -use std::os::unix::io::RawFd; -#[cfg(any(target_os = "linux", target_os = "android"))] -type Result = std::result::Result; - -use crate::error::UResult; - -#[cfg(unix)] -use std::os::unix::io::{AsFd, AsRawFd}; - -#[cfg(unix)] -/// A readable file descriptor. Available in unix and unix-line platforms. -pub trait FdReadable: Read + AsFd + AsRawFd {} -#[cfg(not(unix))] -/// A readable file descriptor. Available in non-unix platforms. -pub trait FdReadable: Read {} - -#[cfg(unix)] -impl FdReadable for T where T: Read + AsFd + AsRawFd {} -#[cfg(not(unix))] -impl FdReadable for T where T: Read {} - -#[cfg(unix)] -/// A writable file descriptor. Available in unix and unix-line platforms. -pub trait FdWritable: Write + AsFd + AsRawFd {} -#[cfg(not(unix))] -/// A writable file descriptor. Available in non-unix platforms. -pub trait FdWritable: Write {} - -#[cfg(unix)] -impl FdWritable for T where T: Write + AsFd + AsRawFd {} -#[cfg(not(unix))] -impl FdWritable for T where T: Write {} - -/// Error types used by buffer-copying functions from the `buf_copy` module. -#[derive(Debug)] -pub enum Error { - Io(std::io::Error), - WriteError(String), -} - -impl std::fmt::Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Error::WriteError(msg) => write!(f, "splice() write error: {}", msg), - Error::Io(err) => write!(f, "I/O error: {}", err), - } - } -} - -impl std::error::Error for Error {} - -impl UError for Error { - fn code(&self) -> i32 { - 1 - } - - fn usage(&self) -> bool { - false - } -} - -// The generalization of this function (and other splice_data functions) is not trivial as most -// utilities will just write data finitely. However, `yes`, which is the sole crate using these -// functions as of now, continuously loops the data write. Coupling the `is_pipe` check together -// with the data write logic means that the check has to be done for every single write, which adds -// unnecessary overhead. -// -/// Helper function to determine whether a given handle (such as a file) is a pipe or not. Can be -/// used to determine whether to use the `splice_data_to_pipe` or the `splice_data_to_fd` function. -/// This function is available exclusively to Linux and Android as it is meant to be used at the -/// scope of splice operations. -/// -/// -/// # Arguments -/// * `out` - path of handle -/// -/// # Returns -/// A `bool` indicating whether the given handle is a pipe or not. -#[inline] -#[cfg(any(target_os = "linux", target_os = "android"))] -pub fn is_pipe

(path: &P) -> Result -where - P: AsRawFd, -{ - Ok(fstat(path.as_raw_fd())?.st_mode as nix::libc::mode_t & S_IFIFO != 0) -} - -#[cfg(any(target_os = "linux", target_os = "android"))] -const SPLICE_SIZE: usize = 1024 * 128; -#[cfg(any(target_os = "linux", target_os = "android"))] -const BUF_SIZE: usize = 1024 * 16; - -/// Copy data from `Read` implementor `source` into a `Write` implementor -/// `dest`. This works by reading a chunk of data from `source` and writing the -/// data to `dest` in a loop. -/// -/// This function uses the Linux-specific `splice` call when possible which does -/// not use any intermediate user-space buffer. It falls backs to -/// `std::io::copy` under other platforms or when the call fails and is still -/// recoverable. -/// -/// # Arguments -/// * `source` - `Read` implementor to copy data from. -/// * `dest` - `Write` implementor to copy data to. -/// -/// # Returns -/// -/// Result of operation and bytes successfully written (as a `u64`) when -/// operation is successful. -pub fn copy_stream(src: &mut R, dest: &mut S) -> UResult -where - R: FdReadable, - S: FdWritable, -{ - #[cfg(any(target_os = "linux", target_os = "android"))] - { - // If we're on Linux or Android, try to use the splice() system call - // for faster writing. If it works, we're done. - let result = splice_write(src, &dest.as_fd())?; - if !result.1 { - return Ok(result.0); - } - } - // If we're not on Linux or Android, or the splice() call failed, - // fall back on slower writing. - let result = std::io::copy(src, dest)?; - - // If the splice() call failed and there has been some data written to - // stdout via while loop above AND there will be second splice() call - // that will succeed, data pushed through splice will be output before - // the data buffered in stdout.lock. Therefore additional explicit flush - // is required here. - dest.flush()?; - Ok(result) -} - -/// Write from source `handle` into destination `write_fd` using Linux-specific -/// `splice` system call. -/// -/// # Arguments -/// - `source` - source handle -/// - `dest` - destination handle -#[inline] -#[cfg(any(target_os = "linux", target_os = "android"))] -fn splice_write(source: &R, dest: &S) -> UResult<(u64, bool)> -where - R: Read + AsFd + AsRawFd, - S: AsRawFd + AsFd, -{ - let (pipe_rd, pipe_wr) = pipe()?; - let mut bytes: u64 = 0; - - loop { - match splice(&source, &pipe_wr, SPLICE_SIZE) { - Ok(n) => { - if n == 0 { - return Ok((bytes, false)); - } - if splice_exact(&pipe_rd, dest, n).is_err() { - // If the first splice manages to copy to the intermediate - // pipe, but the second splice to stdout fails for some reason - // we can recover by copying the data that we have from the - // intermediate pipe to stdout using normal read/write. Then - // we tell the caller to fall back. - copy_exact(pipe_rd.as_raw_fd(), dest, n)?; - return Ok((bytes, true)); - } - - bytes += n as u64; - } - Err(_) => { - return Ok((bytes, true)); - } - } - } -} - -/// Move exactly `num_bytes` bytes from `read_fd` to `write_fd` using the `read` -/// and `write` calls. -#[cfg(any(target_os = "linux", target_os = "android"))] -fn copy_exact(read_fd: RawFd, write_fd: &impl AsFd, num_bytes: usize) -> std::io::Result { - let mut left = num_bytes; - let mut buf = [0; BUF_SIZE]; - let mut written = 0; - while left > 0 { - let read = unistd::read(read_fd, &mut buf)?; - assert_ne!(read, 0, "unexpected end of pipe"); - while written < read { - let n = unistd::write(write_fd, &buf[written..read])?; - written += n; - } - left -= read; - } - Ok(written) -} - -/// Write input `bytes` to a file descriptor. This uses the Linux-specific -/// `vmsplice()` call to write into a file descriptor directly, which only works -/// if the destination is a pipe. -/// -/// # Arguments -/// * `bytes` - data to be written -/// * `dest` - destination handle -/// -/// # Returns -/// When write succeeds, the amount of bytes written is returned as a -/// `u64`. The `bool` indicates if we need to fall back to normal copying or -/// not. `true` means we need to fall back, `false` means we don't have to. -/// -/// A `UError` error is returned when the operation is not supported or when an -/// I/O error occurs. -#[cfg(any(target_os = "linux", target_os = "android"))] -pub fn splice_data_to_pipe(bytes: &[u8], dest: &T) -> UResult<(u64, bool)> -where - T: AsRawFd + AsFd, -{ - let mut n_bytes: u64 = 0; - let mut bytes = bytes; - while !bytes.is_empty() { - let len = match vmsplice(dest, bytes) { - Ok(n) => n, - // The maybe_unsupported call below may emit an error, when the - // error is considered as unrecoverable error (ones that won't make - // us fall back to other method) - Err(e) => return Ok(maybe_unsupported(e)?), - }; - bytes = &bytes[len..]; - n_bytes += len as u64; - } - Ok((n_bytes, false)) -} - -/// Write input `bytes` to a handle using a temporary pipe. A `vmsplice()` call -/// is issued to write to the temporary pipe, which then gets written to the -/// final destination using `splice()`. -/// -/// # Arguments * `bytes` - data to be written * `dest` - destination handle -/// -/// # Returns When write succeeds, the amount of bytes written is returned as a -/// `u64`. The `bool` indicates if we need to fall back to normal copying or -/// not. `true` means we need to fall back, `false` means we don't have to. -/// -/// A `UError` error is returned when the operation is not supported or when an -/// I/O error occurs. -#[cfg(any(target_os = "linux", target_os = "android"))] -pub fn splice_data_to_fd( - bytes: &[u8], - read_pipe: &File, - write_pipe: &File, - dest: &T, -) -> UResult<(u64, bool)> { - let mut n_bytes: u64 = 0; - let mut bytes = bytes; - while !bytes.is_empty() { - let len = match vmsplice(&write_pipe, bytes) { - Ok(n) => n, - Err(e) => return Ok(maybe_unsupported(e)?), - }; - if let Err(e) = splice_exact(&read_pipe, dest, len) { - return Ok(maybe_unsupported(e)?); - } - bytes = &bytes[len..]; - n_bytes += len as u64; - } - Ok((n_bytes, false)) -} - -/// Conversion from a `nix::Error` into our `Error` which implements `UError`. -#[cfg(unix)] -impl From for Error { - fn from(error: nix::Error) -> Self { - Self::Io(std::io::Error::from_raw_os_error(error as i32)) - } -} - -/// Several error values from `nix::Error` (`EINVAL`, `ENOSYS`, and `EBADF`) get -/// treated as errors indicating that the `splice` call is not available, i.e we -/// can still recover from the error. Thus, return the final result of the call -/// as `Result` and indicate that we have to fall back using other write method. -/// -/// # Arguments -/// * `error` - the `nix::Error` received -/// -/// # Returns -/// Result with tuple containing a `u64` `0` indicating that no data had been -/// written and a `true` indicating we have to fall back, if error is still -/// recoverable. Returns an `Error` implementing `UError` otherwise. -#[cfg(any(target_os = "linux", target_os = "android"))] -fn maybe_unsupported(error: nix::Error) -> Result<(u64, bool)> { - match error { - Errno::EINVAL | Errno::ENOSYS | Errno::EBADF => Ok((0, true)), - _ => Err(error.into()), - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::fs::File; - use tempfile::tempdir; - - #[cfg(unix)] - use crate::pipes; - #[cfg(unix)] - use std::fs::OpenOptions; - - #[cfg(unix)] - fn new_temp_file() -> File { - let temp_dir = tempdir().unwrap(); - OpenOptions::new() - .read(true) - .write(true) - .create(true) - .open(temp_dir.path().join("file.txt")) - .unwrap() - } - - #[cfg(any(target_os = "linux", target_os = "android"))] - #[test] - fn test_file_is_pipe() { - let temp_file = new_temp_file(); - let (pipe_read, pipe_write) = pipes::pipe().unwrap(); - - assert!(is_pipe(&pipe_read).unwrap()); - assert!(is_pipe(&pipe_write).unwrap()); - assert!(!is_pipe(&temp_file).unwrap()); - } - - #[cfg(any(target_os = "linux", target_os = "android"))] - #[test] - fn test_valid_splice_errs() { - use nix::errno::Errno; - use nix::Error; - - let err = Error::from(Errno::EINVAL); - assert_eq!(maybe_unsupported(err).unwrap(), (0, true)); - - let err = Error::from(Errno::ENOSYS); - assert_eq!(maybe_unsupported(err).unwrap(), (0, true)); - - let err = Error::from(Errno::EBADF); - assert_eq!(maybe_unsupported(err).unwrap(), (0, true)); - - let err = Error::from(Errno::EPERM); - assert!(maybe_unsupported(err).is_err()); - } - - #[cfg(any(target_os = "linux", target_os = "android"))] - #[test] - fn test_splice_data_to_pipe() { - let (pipe_read, pipe_write) = pipes::pipe().unwrap(); - let data = b"Hello, world!"; - let (bytes, _) = splice_data_to_pipe(data, &pipe_write).unwrap(); - let mut buf = [0; 1024]; - let n = unistd::read(pipe_read.as_raw_fd(), &mut buf).unwrap(); - assert_eq!(&buf[..n], data); - assert_eq!(bytes as usize, data.len()); - } - - #[cfg(any(target_os = "linux", target_os = "android"))] - #[test] - fn test_splice_data_to_file() { - use std::io::{Seek, SeekFrom}; - - let mut temp_file = new_temp_file(); - let (pipe_read, pipe_write) = pipes::pipe().unwrap(); - let data = b"Hello, world!"; - let (bytes, _) = splice_data_to_fd(data, &pipe_read, &pipe_write, &temp_file).unwrap(); - assert_eq!(bytes as usize, data.len()); - - // We would have been at the end already, so seek again to the start. - temp_file.seek(SeekFrom::Start(0)).unwrap(); - - let mut buf = Vec::new(); - temp_file.read_to_end(&mut buf).unwrap(); - assert_eq!(buf, data); - } - - #[cfg(any(target_os = "linux", target_os = "android"))] - #[test] - fn test_copy_exact() { - let (mut pipe_read, mut pipe_write) = pipes::pipe().unwrap(); - let data = b"Hello, world!"; - let n = pipe_write.write(data).unwrap(); - assert_eq!(n, data.len()); - let mut buf = [0; 1024]; - let n = copy_exact(pipe_read.as_raw_fd(), &pipe_write, data.len()).unwrap(); - let n2 = pipe_read.read(&mut buf).unwrap(); - assert_eq!(n, n2); - assert_eq!(&buf[..n], data); - } - - #[test] - #[cfg(unix)] - fn test_copy_stream() { - use std::{ - io::{Seek, SeekFrom}, - thread, - }; - let mut dest_file = new_temp_file(); - - let (mut pipe_read, mut pipe_write) = pipes::pipe().unwrap(); - let data = b"Hello, world!"; - let thread = thread::spawn(move || { - pipe_write.write_all(data).unwrap(); - }); - let result = copy_stream(&mut pipe_read, &mut dest_file).unwrap(); - thread.join().unwrap(); - assert!(result == data.len() as u64); - - // We would have been at the end already, so seek again to the start. - dest_file.seek(SeekFrom::Start(0)).unwrap(); - - let mut buf = Vec::new(); - dest_file.read_to_end(&mut buf).unwrap(); - - assert_eq!(buf, data); - } - - #[test] - #[cfg(not(unix))] - // Test for non-unix platforms. We use regular files instead. - fn test_copy_stream() { - let temp_dir = tempdir().unwrap(); - let src_path = temp_dir.path().join("src.txt"); - let dest_path = temp_dir.path().join("dest.txt"); - - let mut src_file = File::create(&src_path).unwrap(); - let mut dest_file = File::create(&dest_path).unwrap(); - - let data = b"Hello, world!"; - src_file.write_all(data).unwrap(); - src_file.sync_all().unwrap(); - - let mut src_file = File::open(&src_path).unwrap(); - let bytes_copied = copy_stream(&mut src_file, &mut dest_file).unwrap(); - - let mut dest_file = File::open(&dest_path).unwrap(); - let mut buf = Vec::new(); - dest_file.read_to_end(&mut buf).unwrap(); - - assert_eq!(bytes_copied as usize, data.len()); - assert_eq!(buf, data); - } - - #[cfg(any(target_os = "linux", target_os = "android"))] - #[test] - fn test_splice_write() { - use std::{ - io::{Seek, SeekFrom}, - thread, - }; - - let (pipe_read, mut pipe_write) = pipes::pipe().unwrap(); - let mut dest_file = new_temp_file(); - let data = b"Hello, world!"; - let thread = thread::spawn(move || { - pipe_write.write_all(data).unwrap(); - }); - let (bytes, _) = splice_write(&pipe_read, &dest_file).unwrap(); - thread.join().unwrap(); - - assert!(bytes == data.len() as u64); - - // We would have been at the end already, so seek again to the start. - dest_file.seek(SeekFrom::Start(0)).unwrap(); - - let mut buf = Vec::new(); - dest_file.read_to_end(&mut buf).unwrap(); +pub use linux::*; - assert_eq!(buf, data); - } -} +#[cfg(not(any(target_os = "linux", target_os = "android")))] +pub mod other; +#[cfg(not(any(target_os = "linux", target_os = "android")))] +pub use other::copy_stream; diff --git a/src/uucore/src/lib/features/buf_copy/common.rs b/src/uucore/src/lib/features/buf_copy/common.rs new file mode 100644 index 00000000000..2538755fc92 --- /dev/null +++ b/src/uucore/src/lib/features/buf_copy/common.rs @@ -0,0 +1,29 @@ +use crate::error::UError; + +/// Error types used by buffer-copying functions from the `buf_copy` module. +#[derive(Debug)] +pub enum Error { + Io(std::io::Error), + WriteError(String), +} + +impl std::fmt::Display for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Error::WriteError(msg) => write!(f, "splice() write error: {}", msg), + Error::Io(err) => write!(f, "I/O error: {}", err), + } + } +} + +impl std::error::Error for Error {} + +impl UError for Error { + fn code(&self) -> i32 { + 1 + } + + fn usage(&self) -> bool { + false + } +} diff --git a/src/uucore/src/lib/features/buf_copy/linux.rs b/src/uucore/src/lib/features/buf_copy/linux.rs new file mode 100644 index 00000000000..2a8026987ed --- /dev/null +++ b/src/uucore/src/lib/features/buf_copy/linux.rs @@ -0,0 +1,255 @@ +use nix::sys::stat::fstat; +use nix::{errno::Errno, libc::S_IFIFO}; + +type Result = std::result::Result; + +/// Buffer-based copying utilities for Linux and Android. +use crate::{ + error::UResult, + pipes::{pipe, splice, splice_exact, vmsplice}, +}; + +/// Buffer-based copying utilities for unix (excluding Linux). +use std::{ + fs::File, + io::{Read, Write}, + os::fd::{AsFd, AsRawFd, RawFd}, +}; + +use super::common::Error; + +/// A readable file descriptor. +pub trait FdReadable: Read + AsRawFd + AsFd {} + +impl FdReadable for T where T: Read + AsFd + AsRawFd {} + +/// A writable file descriptor. +pub trait FdWritable: Write + AsFd + AsRawFd {} + +impl FdWritable for T where T: Write + AsFd + AsRawFd {} + +const SPLICE_SIZE: usize = 1024 * 128; +const BUF_SIZE: usize = 1024 * 16; + +/// Conversion from a `nix::Error` into our `Error` which implements `UError`. +impl From for Error { + fn from(error: nix::Error) -> Self { + Self::Io(std::io::Error::from_raw_os_error(error as i32)) + } +} + +/// Copy data from `Read` implementor `source` into a `Write` implementor +/// `dest`. This works by reading a chunk of data from `source` and writing the +/// data to `dest` in a loop. +/// +/// This function uses the Linux-specific `splice` call when possible which does +/// not use any intermediate user-space buffer. It falls backs to +/// `std::io::copy` when the call fails and is still recoverable. +/// +/// # Arguments * `source` - `Read` implementor to copy data from. * `dest` - +/// `Write` implementor to copy data to. +/// +/// # Returns +/// +/// Result of operation and bytes successfully written (as a `u64`) when +/// operation is successful. +/// + +pub fn copy_stream(src: &mut R, dest: &mut S) -> UResult +where + R: Read + AsFd + AsRawFd, + S: Write + AsFd + AsRawFd, +{ + // If we're on Linux or Android, try to use the splice() system call + // for faster writing. If it works, we're done. + let result = splice_write(src, &dest.as_fd())?; + if !result.1 { + return Ok(result.0); + } + + // If the splice() call failed, fall back on slower writing. + let result = std::io::copy(src, dest)?; + + // If the splice() call failed and there has been some data written to + // stdout via while loop above AND there will be second splice() call + // that will succeed, data pushed through splice will be output before + // the data buffered in stdout.lock. Therefore additional explicit flush + // is required here. + dest.flush()?; + Ok(result) +} + +/// Write from source `handle` into destination `write_fd` using Linux-specific +/// `splice` system call. +/// +/// # Arguments +/// - `source` - source handle +/// - `dest` - destination handle +#[inline] +fn splice_write(source: &R, dest: &S) -> UResult<(u64, bool)> +where + R: Read + AsFd + AsRawFd, + S: AsRawFd + AsFd, +{ + let (pipe_rd, pipe_wr) = pipe()?; + let mut bytes: u64 = 0; + + loop { + match splice(&source, &pipe_wr, SPLICE_SIZE) { + Ok(n) => { + if n == 0 { + return Ok((bytes, false)); + } + if splice_exact(&pipe_rd, dest, n).is_err() { + // If the first splice manages to copy to the intermediate + // pipe, but the second splice to stdout fails for some reason + // we can recover by copying the data that we have from the + // intermediate pipe to stdout using normal read/write. Then + // we tell the caller to fall back. + copy_exact(pipe_rd.as_raw_fd(), dest, n)?; + return Ok((bytes, true)); + } + + bytes += n as u64; + } + Err(_) => { + return Ok((bytes, true)); + } + } + } +} + +/// Move exactly `num_bytes` bytes from `read_fd` to `write_fd` using the `read` +/// and `write` calls. +#[cfg(any(target_os = "linux", target_os = "android"))] +fn copy_exact(read_fd: RawFd, write_fd: &impl AsFd, num_bytes: usize) -> std::io::Result { + use nix::unistd; + + let mut left = num_bytes; + let mut buf = [0; BUF_SIZE]; + let mut written = 0; + while left > 0 { + let read = unistd::read(read_fd, &mut buf)?; + assert_ne!(read, 0, "unexpected end of pipe"); + while written < read { + let n = unistd::write(write_fd, &buf[written..read])?; + written += n; + } + left -= read; + } + Ok(written) +} + +// The generalization of this function (and other splice_data functions) is not trivial as most +// utilities will just write data finitely. However, `yes`, which is the sole crate using these +// functions as of now, continuously loops the data write. Coupling the `is_pipe` check together +// with the data write logic means that the check has to be done for every single write, which adds +// unnecessary overhead. +// +/// Helper function to determine whether a given handle (such as a file) is a pipe or not. Can be +/// used to determine whether to use the `splice_data_to_pipe` or the `splice_data_to_fd` function. +/// This function is available exclusively to Linux and Android as it is meant to be used at the +/// scope of splice operations. +/// +/// +/// # Arguments +/// * `out` - path of handle +/// +/// # Returns +/// A `bool` indicating whether the given handle is a pipe or not. +#[inline] +pub fn is_pipe

(path: &P) -> Result +where + P: AsRawFd, +{ + Ok(fstat(path.as_raw_fd())?.st_mode as nix::libc::mode_t & S_IFIFO != 0) +} + +/// Write input `bytes` to a handle using a temporary pipe. A `vmsplice()` call +/// is issued to write to the temporary pipe, which then gets written to the +/// final destination using `splice()`. +/// +/// # Arguments * `bytes` - data to be written * `dest` - destination handle +/// +/// # Returns When write succeeds, the amount of bytes written is returned as a +/// `u64`. The `bool` indicates if we need to fall back to normal copying or +/// not. `true` means we need to fall back, `false` means we don't have to. +/// +/// A `UError` error is returned when the operation is not supported or when an +/// I/O error occurs. +pub fn splice_data_to_fd( + bytes: &[u8], + read_pipe: &File, + write_pipe: &File, + dest: &T, +) -> UResult<(u64, bool)> { + let mut n_bytes: u64 = 0; + let mut bytes = bytes; + while !bytes.is_empty() { + let len = match vmsplice(&write_pipe, bytes) { + Ok(n) => n, + Err(e) => return Ok(maybe_unsupported(e)?), + }; + if let Err(e) = splice_exact(&read_pipe, dest, len) { + return Ok(maybe_unsupported(e)?); + } + bytes = &bytes[len..]; + n_bytes += len as u64; + } + Ok((n_bytes, false)) +} + +/// Write input `bytes` to a file descriptor. This uses the Linux-specific +/// `vmsplice()` call to write into a file descriptor directly, which only works +/// if the destination is a pipe. +/// +/// # Arguments +/// * `bytes` - data to be written +/// * `dest` - destination handle +/// +/// # Returns +/// When write succeeds, the amount of bytes written is returned as a +/// `u64`. The `bool` indicates if we need to fall back to normal copying or +/// not. `true` means we need to fall back, `false` means we don't have to. +/// +/// A `UError` error is returned when the operation is not supported or when an +/// I/O error occurs. +#[cfg(any(target_os = "linux", target_os = "android"))] +pub fn splice_data_to_pipe(bytes: &[u8], dest: &T) -> UResult<(u64, bool)> +where + T: AsRawFd + AsFd, +{ + let mut n_bytes: u64 = 0; + let mut bytes = bytes; + while !bytes.is_empty() { + let len = match vmsplice(dest, bytes) { + Ok(n) => n, + // The maybe_unsupported call below may emit an error, when the + // error is considered as unrecoverable error (ones that won't make + // us fall back to other method) + Err(e) => return Ok(maybe_unsupported(e)?), + }; + bytes = &bytes[len..]; + n_bytes += len as u64; + } + Ok((n_bytes, false)) +} + +/// Several error values from `nix::Error` (`EINVAL`, `ENOSYS`, and `EBADF`) get +/// treated as errors indicating that the `splice` call is not available, i.e we +/// can still recover from the error. Thus, return the final result of the call +/// as `Result` and indicate that we have to fall back using other write method. +/// +/// # Arguments +/// * `error` - the `nix::Error` received +/// +/// # Returns +/// Result with tuple containing a `u64` `0` indicating that no data had been +/// written and a `true` indicating we have to fall back, if error is still +/// recoverable. Returns an `Error` implementing `UError` otherwise. +fn maybe_unsupported(error: nix::Error) -> Result<(u64, bool)> { + match error { + Errno::EINVAL | Errno::ENOSYS | Errno::EBADF => Ok((0, true)), + _ => Err(error.into()), + } +} diff --git a/src/uucore/src/lib/features/buf_copy/other.rs b/src/uucore/src/lib/features/buf_copy/other.rs new file mode 100644 index 00000000000..6602ca5ab11 --- /dev/null +++ b/src/uucore/src/lib/features/buf_copy/other.rs @@ -0,0 +1,25 @@ +use std::io::{Read, Write}; + +use crate::error::UResult; + +/// Copy data from `Read` implementor `source` into a `Write` implementor +/// `dest`. This works by reading a chunk of data from `source` and writing the +/// data to `dest` in a loop, using std::io::copy. This is implemented for +/// non-Linux platforms. +/// +/// # Arguments +/// * `source` - `Read` implementor to copy data from. +/// * `dest` - `Write` implementor to copy data to. +/// +/// # Returns +/// +/// Result of operation and bytes successfully written (as a `u64`) when +/// operation is successful. +pub fn copy_stream(src: &mut R, dest: &mut S) -> UResult +where + R: Read, + S: Write, +{ + let result = std::io::copy(src, dest)?; + Ok(result) +} From 148c531f6cc02baec8af74d0eaf128b0c9e85935 Mon Sep 17 00:00:00 2001 From: Daringcuteseal Date: Mon, 23 Dec 2024 07:55:13 +0700 Subject: [PATCH 5/7] uucore/buf_copy: add tests --- src/uucore/src/lib/features/buf_copy.rs | 187 ++++++++++++++++++ src/uucore/src/lib/features/buf_copy/linux.rs | 10 +- 2 files changed, 194 insertions(+), 3 deletions(-) diff --git a/src/uucore/src/lib/features/buf_copy.rs b/src/uucore/src/lib/features/buf_copy.rs index 5ea5d8eaa12..f33ce5b24e2 100644 --- a/src/uucore/src/lib/features/buf_copy.rs +++ b/src/uucore/src/lib/features/buf_copy.rs @@ -20,3 +20,190 @@ pub use linux::*; pub mod other; #[cfg(not(any(target_os = "linux", target_os = "android")))] pub use other::copy_stream; + +#[cfg(test)] +mod tests { + use super::*; + use std::fs::File; + use tempfile::tempdir; + + #[cfg(unix)] + use crate::pipes; + #[cfg(unix)] + use std::fs::OpenOptions; + #[cfg(unix)] + use std::{ + io::{Seek, SeekFrom}, + thread, + }; + + #[cfg(any(target_os = "linux", target_os = "android"))] + use nix::unistd; + #[cfg(any(target_os = "linux", target_os = "android"))] + use std::os::fd::AsRawFd; + + use std::io::{Read, Write}; + + #[cfg(unix)] + fn new_temp_file() -> File { + let temp_dir = tempdir().unwrap(); + OpenOptions::new() + .read(true) + .write(true) + .create(true) + .open(temp_dir.path().join("file.txt")) + .unwrap() + } + + #[cfg(any(target_os = "linux", target_os = "android"))] + #[test] + fn test_file_is_pipe() { + let temp_file = new_temp_file(); + let (pipe_read, pipe_write) = pipes::pipe().unwrap(); + + assert!(is_pipe(&pipe_read).unwrap()); + assert!(is_pipe(&pipe_write).unwrap()); + assert!(!is_pipe(&temp_file).unwrap()); + } + + #[cfg(any(target_os = "linux", target_os = "android"))] + #[test] + fn test_valid_splice_errs() { + use nix::errno::Errno; + use nix::Error; + + let err = Error::from(Errno::EINVAL); + assert_eq!(maybe_unsupported(err).unwrap(), (0, true)); + + let err = Error::from(Errno::ENOSYS); + assert_eq!(maybe_unsupported(err).unwrap(), (0, true)); + + let err = Error::from(Errno::EBADF); + assert_eq!(maybe_unsupported(err).unwrap(), (0, true)); + + let err = Error::from(Errno::EPERM); + assert!(maybe_unsupported(err).is_err()); + } + + #[cfg(any(target_os = "linux", target_os = "android"))] + #[test] + fn test_splice_data_to_pipe() { + let (pipe_read, pipe_write) = pipes::pipe().unwrap(); + let data = b"Hello, world!"; + let (bytes, _) = splice_data_to_pipe(data, &pipe_write).unwrap(); + let mut buf = [0; 1024]; + let n = unistd::read(pipe_read.as_raw_fd(), &mut buf).unwrap(); + assert_eq!(&buf[..n], data); + assert_eq!(bytes as usize, data.len()); + } + + #[cfg(any(target_os = "linux", target_os = "android"))] + #[test] + fn test_splice_data_to_file() { + use std::io::{Read, Seek, SeekFrom}; + + let mut temp_file = new_temp_file(); + let (pipe_read, pipe_write) = pipes::pipe().unwrap(); + let data = b"Hello, world!"; + let (bytes, _) = splice_data_to_fd(data, &pipe_read, &pipe_write, &temp_file).unwrap(); + assert_eq!(bytes as usize, data.len()); + + // We would have been at the end already, so seek again to the start. + temp_file.seek(SeekFrom::Start(0)).unwrap(); + + let mut buf = Vec::new(); + temp_file.read_to_end(&mut buf).unwrap(); + assert_eq!(buf, data); + } + + #[cfg(any(target_os = "linux", target_os = "android"))] + #[test] + fn test_copy_exact() { + let (mut pipe_read, mut pipe_write) = pipes::pipe().unwrap(); + let data = b"Hello, world!"; + let n = pipe_write.write(data).unwrap(); + assert_eq!(n, data.len()); + let mut buf = [0; 1024]; + let n = copy_exact(pipe_read.as_raw_fd(), &pipe_write, data.len()).unwrap(); + let n2 = pipe_read.read(&mut buf).unwrap(); + assert_eq!(n, n2); + assert_eq!(&buf[..n], data); + } + + #[test] + #[cfg(unix)] + fn test_copy_stream() { + let mut dest_file = new_temp_file(); + + let (mut pipe_read, mut pipe_write) = pipes::pipe().unwrap(); + let data = b"Hello, world!"; + let thread = thread::spawn(move || { + pipe_write.write_all(data).unwrap(); + }); + let result = copy_stream(&mut pipe_read, &mut dest_file).unwrap(); + thread.join().unwrap(); + assert!(result == data.len() as u64); + + // We would have been at the end already, so seek again to the start. + dest_file.seek(SeekFrom::Start(0)).unwrap(); + + let mut buf = Vec::new(); + dest_file.read_to_end(&mut buf).unwrap(); + + assert_eq!(buf, data); + } + + #[test] + #[cfg(not(unix))] + // Test for non-unix platforms. We use regular files instead. + fn test_copy_stream() { + let temp_dir = tempdir().unwrap(); + let src_path = temp_dir.path().join("src.txt"); + let dest_path = temp_dir.path().join("dest.txt"); + + let mut src_file = File::create(&src_path).unwrap(); + let mut dest_file = File::create(&dest_path).unwrap(); + + let data = b"Hello, world!"; + src_file.write_all(data).unwrap(); + src_file.sync_all().unwrap(); + + let mut src_file = File::open(&src_path).unwrap(); + let bytes_copied = copy_stream(&mut src_file, &mut dest_file).unwrap(); + + let mut dest_file = File::open(&dest_path).unwrap(); + let mut buf = Vec::new(); + dest_file.read_to_end(&mut buf).unwrap(); + + assert_eq!(bytes_copied as usize, data.len()); + assert_eq!(buf, data); + } + + #[cfg(any(target_os = "linux", target_os = "android"))] + #[test] + fn test_splice_write() { + use std::{ + io::{Read, Seek, SeekFrom, Write}, + thread, + }; + + let (pipe_read, mut pipe_write) = pipes::pipe().unwrap(); + let mut dest_file = new_temp_file(); + let data = b"Hello, world!"; + let thread = thread::spawn(move || { + pipe_write.write_all(data).unwrap(); + }); + let (bytes, _) = splice_write(&pipe_read, &dest_file).unwrap(); + thread.join().unwrap(); + + assert!(bytes == data.len() as u64); + + // We would have been at the end already, so seek again to the start. + dest_file.seek(SeekFrom::Start(0)).unwrap(); + + let mut buf = Vec::new(); + dest_file.read_to_end(&mut buf).unwrap(); + + assert_eq!(buf, data); + } +} diff --git a/src/uucore/src/lib/features/buf_copy/linux.rs b/src/uucore/src/lib/features/buf_copy/linux.rs index 2a8026987ed..247b827a943 100644 --- a/src/uucore/src/lib/features/buf_copy/linux.rs +++ b/src/uucore/src/lib/features/buf_copy/linux.rs @@ -86,7 +86,7 @@ where /// - `source` - source handle /// - `dest` - destination handle #[inline] -fn splice_write(source: &R, dest: &S) -> UResult<(u64, bool)> +pub(crate) fn splice_write(source: &R, dest: &S) -> UResult<(u64, bool)> where R: Read + AsFd + AsRawFd, S: AsRawFd + AsFd, @@ -122,7 +122,11 @@ where /// Move exactly `num_bytes` bytes from `read_fd` to `write_fd` using the `read` /// and `write` calls. #[cfg(any(target_os = "linux", target_os = "android"))] -fn copy_exact(read_fd: RawFd, write_fd: &impl AsFd, num_bytes: usize) -> std::io::Result { +pub(crate) fn copy_exact( + read_fd: RawFd, + write_fd: &impl AsFd, + num_bytes: usize, +) -> std::io::Result { use nix::unistd; let mut left = num_bytes; @@ -247,7 +251,7 @@ where /// Result with tuple containing a `u64` `0` indicating that no data had been /// written and a `true` indicating we have to fall back, if error is still /// recoverable. Returns an `Error` implementing `UError` otherwise. -fn maybe_unsupported(error: nix::Error) -> Result<(u64, bool)> { +pub(crate) fn maybe_unsupported(error: nix::Error) -> Result<(u64, bool)> { match error { Errno::EINVAL | Errno::ENOSYS | Errno::EBADF => Ok((0, true)), _ => Err(error.into()), From f6b963486d90a6b385ab20fc1197018a5f97c85e Mon Sep 17 00:00:00 2001 From: Pearl Date: Sat, 28 Dec 2024 19:59:47 +0700 Subject: [PATCH 6/7] uucore/buf_copy: simplify unix and linux-specific imports Co-authored-by: Sylvestre Ledru --- src/uucore/src/lib/features/buf_copy.rs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/uucore/src/lib/features/buf_copy.rs b/src/uucore/src/lib/features/buf_copy.rs index f33ce5b24e2..d82f8d4d1c3 100644 --- a/src/uucore/src/lib/features/buf_copy.rs +++ b/src/uucore/src/lib/features/buf_copy.rs @@ -28,19 +28,17 @@ mod tests { use tempfile::tempdir; #[cfg(unix)] - use crate::pipes; - #[cfg(unix)] - use std::fs::OpenOptions; - #[cfg(unix)] - use std::{ - io::{Seek, SeekFrom}, - thread, + use { + crate::pipes, + std::fs::OpenOptions, + std::{ + io::{Seek, SeekFrom}, + thread, + }, }; #[cfg(any(target_os = "linux", target_os = "android"))] - use nix::unistd; - #[cfg(any(target_os = "linux", target_os = "android"))] - use std::os::fd::AsRawFd; + use {nix::unistd, std::os::fd::AsRawFd}; use std::io::{Read, Write}; From c2f3a762a8557dadda57373591dea75553a54193 Mon Sep 17 00:00:00 2001 From: Daringcuteseal Date: Sat, 28 Dec 2024 20:29:07 +0700 Subject: [PATCH 7/7] uucore/buf_copy: add copyright notice --- src/uucore/src/lib/features/buf_copy/common.rs | 5 +++++ src/uucore/src/lib/features/buf_copy/linux.rs | 5 +++++ src/uucore/src/lib/features/buf_copy/other.rs | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/src/uucore/src/lib/features/buf_copy/common.rs b/src/uucore/src/lib/features/buf_copy/common.rs index 2538755fc92..8c74dbb8a88 100644 --- a/src/uucore/src/lib/features/buf_copy/common.rs +++ b/src/uucore/src/lib/features/buf_copy/common.rs @@ -1,3 +1,8 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + use crate::error::UError; /// Error types used by buffer-copying functions from the `buf_copy` module. diff --git a/src/uucore/src/lib/features/buf_copy/linux.rs b/src/uucore/src/lib/features/buf_copy/linux.rs index 247b827a943..77d25e44beb 100644 --- a/src/uucore/src/lib/features/buf_copy/linux.rs +++ b/src/uucore/src/lib/features/buf_copy/linux.rs @@ -1,3 +1,8 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + use nix::sys::stat::fstat; use nix::{errno::Errno, libc::S_IFIFO}; diff --git a/src/uucore/src/lib/features/buf_copy/other.rs b/src/uucore/src/lib/features/buf_copy/other.rs index 6602ca5ab11..6497c9224c6 100644 --- a/src/uucore/src/lib/features/buf_copy/other.rs +++ b/src/uucore/src/lib/features/buf_copy/other.rs @@ -1,3 +1,8 @@ +// This file is part of the uutils coreutils package. +// +// For the full copyright and license information, please view the LICENSE +// file that was distributed with this source code. + use std::io::{Read, Write}; use crate::error::UResult;