From 925842756f5e124a81447acf4d989d2d8929f108 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Fri, 19 Dec 2025 23:23:34 +0000 Subject: [PATCH 1/7] chore(deps): update rust crate crc-fast to v1.8.2 --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 585c2bb1ad5..37b3362e612 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -699,9 +699,9 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc-fast" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c15e7f62c7d6e256e6d0fc3fc1ef395348e4bc395dcf14d6990da0e5aa6e8b0" +checksum = "85d9be5297a59f1b7651fd2711a1f4461929f53b182b394df0df15b3a387ef51" dependencies = [ "crc", "digest", From 9c80a1d93f42c819a290368d60fed8bcbd9f6ae7 Mon Sep 17 00:00:00 2001 From: mattsu <35655889+mattsu2020@users.noreply.github.com> Date: Sat, 20 Dec 2025 17:35:53 +0900 Subject: [PATCH 2/7] fix(sort): GNU sort-continue.sh test (#9107) * feat: dynamically adjust merge batch size based on file descriptor limits - Add `effective_merge_batch_size()` function to calculate batch size considering fd soft limit, with minimums and safety margins. - Generalize fd limit handling from Linux-only to Unix systems using `fd_soft_limit()`. - Update merge logic to use dynamic batch size instead of fixed `settings.merge_batch_size` to prevent fd exhaustion. * fix(sort): update rlimit fetching to use fd_soft_limit with error handling Replace direct call to get_rlimit()? with fd_soft_limit(), adding a check for None value to return a usage error if rlimit cannot be fetched. This improves robustness on Linux by ensuring proper error handling when retrieving the file descriptor soft limit. * refactor(sort): restrict nix::libc and fd_soft_limit to Linux Update conditional compilation attributes from #[cfg(unix)] to #[cfg(target_os = "linux")] for the nix::libc import and fd_soft_limit function implementations, ensuring these features are only enabled on Linux systems to improve portability and avoid issues on other Unix-like platforms. * refactor: improve thread management and replace unsafe libc calls Replace unsafe libc::getrlimit calls in fd_soft_limit with safe nix crate usage. Update Rayon thread configuration to use ThreadPoolBuilder instead of environment variables for better control. Add documentation comment to effective_merge_batch_size function for clarity. * refactor(linux): improve error handling in fd_soft_limit function Extract the rlimit fetching logic into a separate `get_rlimit` function that returns `UResult` and properly handles errors with `UUsageError`, instead of silently returning `None` on failure or infinity. This provides better error reporting for resource limit issues on Linux platforms. * refactor(sort): reorder imports in get_rlimit for consistency Reordered the nix::sys::resource imports to group constants first (RLIM_INFINITY), then types (Resource), and finally functions (getrlimit), improving code readability and adhering to import style guidelines. --- src/uu/sort/src/merge.rs | 39 ++++++++++++++++++++++++++------ src/uu/sort/src/sort.rs | 48 +++++++++++++++++++++++++++++++--------- 2 files changed, 70 insertions(+), 17 deletions(-) diff --git a/src/uu/sort/src/merge.rs b/src/uu/sort/src/merge.rs index ea212f62f34..502dcda82a6 100644 --- a/src/uu/sort/src/merge.rs +++ b/src/uu/sort/src/merge.rs @@ -30,7 +30,7 @@ use uucore::error::{FromIo, UResult}; use crate::{ GlobalSettings, Output, SortError, chunks::{self, Chunk, RecycledChunk}, - compare_by, open, + compare_by, fd_soft_limit, open, tmp_dir::TmpDirWrapper, }; @@ -62,6 +62,28 @@ fn replace_output_file_in_input_files( Ok(()) } +/// Determine the effective merge batch size, enforcing a minimum and respecting the +/// file-descriptor soft limit after reserving stdio/output and a safety margin. +fn effective_merge_batch_size(settings: &GlobalSettings) -> usize { + const MIN_BATCH_SIZE: usize = 2; + const RESERVED_STDIO: usize = 3; + const RESERVED_OUTPUT: usize = 1; + const SAFETY_MARGIN: usize = 1; + let mut batch_size = settings.merge_batch_size.max(MIN_BATCH_SIZE); + + if let Some(limit) = fd_soft_limit() { + let reserved = RESERVED_STDIO + RESERVED_OUTPUT + SAFETY_MARGIN; + let available_inputs = limit.saturating_sub(reserved); + if available_inputs >= MIN_BATCH_SIZE { + batch_size = batch_size.min(available_inputs); + } else { + batch_size = MIN_BATCH_SIZE; + } + } + + batch_size +} + /// Merge pre-sorted `Box`s. /// /// If `settings.merge_batch_size` is greater than the length of `files`, intermediate files will be used. @@ -94,18 +116,21 @@ pub fn merge_with_file_limit< output: Output, tmp_dir: &mut TmpDirWrapper, ) -> UResult<()> { - if files.len() <= settings.merge_batch_size { + let batch_size = effective_merge_batch_size(settings); + debug_assert!(batch_size >= 2); + + if files.len() <= batch_size { let merger = merge_without_limit(files, settings); merger?.write_all(settings, output) } else { let mut temporary_files = vec![]; - let mut batch = vec![]; + let mut batch = Vec::with_capacity(batch_size); for file in files { batch.push(file); - if batch.len() >= settings.merge_batch_size { - assert_eq!(batch.len(), settings.merge_batch_size); + if batch.len() >= batch_size { + assert_eq!(batch.len(), batch_size); let merger = merge_without_limit(batch.into_iter(), settings)?; - batch = vec![]; + batch = Vec::with_capacity(batch_size); let mut tmp_file = Tmp::create(tmp_dir.next_file()?, settings.compress_prog.as_deref())?; @@ -115,7 +140,7 @@ pub fn merge_with_file_limit< } // Merge any remaining files that didn't get merged in a full batch above. if !batch.is_empty() { - assert!(batch.len() < settings.merge_batch_size); + assert!(batch.len() < batch_size); let merger = merge_without_limit(batch.into_iter(), settings)?; let mut tmp_file = diff --git a/src/uu/sort/src/sort.rs b/src/uu/sort/src/sort.rs index 3b967d042ae..6122089e2f3 100644 --- a/src/uu/sort/src/sort.rs +++ b/src/uu/sort/src/sort.rs @@ -1073,13 +1073,27 @@ fn make_sort_mode_arg(mode: &'static str, short: char, help: String) -> Arg { #[cfg(target_os = "linux")] fn get_rlimit() -> UResult { - use nix::sys::resource::{Resource, getrlimit}; + use nix::sys::resource::{RLIM_INFINITY, Resource, getrlimit}; - getrlimit(Resource::RLIMIT_NOFILE) - .map(|(rlim_cur, _)| rlim_cur as usize) + let (rlim_cur, _rlim_max) = getrlimit(Resource::RLIMIT_NOFILE) + .map_err(|_| UUsageError::new(2, translate!("sort-failed-fetch-rlimit")))?; + if rlim_cur == RLIM_INFINITY { + return Err(UUsageError::new(2, translate!("sort-failed-fetch-rlimit"))); + } + usize::try_from(rlim_cur) .map_err(|_| UUsageError::new(2, translate!("sort-failed-fetch-rlimit"))) } +#[cfg(target_os = "linux")] +pub(crate) fn fd_soft_limit() -> Option { + get_rlimit().ok() +} + +#[cfg(not(target_os = "linux"))] +pub(crate) fn fd_soft_limit() -> Option { + None +} + const STDIN_FILE: &str = "-"; /// Legacy `+POS1 [-POS2]` syntax is permitted unless `_POSIX2_VERSION` is in @@ -1232,12 +1246,12 @@ fn default_merge_batch_size() -> usize { #[cfg(target_os = "linux")] { // Adjust merge batch size dynamically based on available file descriptors. - match get_rlimit() { - Ok(limit) => { + match fd_soft_limit() { + Some(limit) => { let usable_limit = limit.saturating_div(LINUX_BATCH_DIVISOR); usable_limit.clamp(LINUX_BATCH_MIN, LINUX_BATCH_MAX) } - Err(_) => 64, + None => 64, } } @@ -1366,9 +1380,15 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { settings.threads = matches .get_one::(options::PARALLEL) .map_or_else(|| "0".to_string(), String::from); - unsafe { - env::set_var("RAYON_NUM_THREADS", &settings.threads); - } + let num_threads = match settings.threads.parse::() { + Ok(0) | Err(_) => std::thread::available_parallelism() + .map(|n| n.get()) + .unwrap_or(1), + Ok(n) => n, + }; + let _ = rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build_global(); } if let Some(size_str) = matches.get_one::(options::BUF_SIZE) { @@ -1419,7 +1439,15 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> { translate!( "sort-maximum-batch-size-rlimit", - "rlimit" => get_rlimit()? + "rlimit" => { + let Some(rlimit) = fd_soft_limit() else { + return Err(UUsageError::new( + 2, + translate!("sort-failed-fetch-rlimit"), + )); + }; + rlimit + } ) } #[cfg(not(target_os = "linux"))] From 375c67c9be430b9dcf0bb02f1ee04e49de6950a8 Mon Sep 17 00:00:00 2001 From: oech3 <79379754+oech3@users.noreply.github.com> Date: Sat, 20 Dec 2025 18:41:57 +0900 Subject: [PATCH 3/7] run-gnu-test.sh: Fix nproc broken by cache (#9735) --- util/run-gnu-test.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/util/run-gnu-test.sh b/util/run-gnu-test.sh index 6d0edee5f14..23d78ca625a 100755 --- a/util/run-gnu-test.sh +++ b/util/run-gnu-test.sh @@ -28,7 +28,8 @@ echo "path_UUTILS='${path_UUTILS}'" echo "path_GNU='${path_GNU}'" # Use GNU nproc for *BSD -MAKEFLAGS="${MAKEFLAGS} -j $(${path_GNU}/src/nproc)" +NPROC=$(command -v ${path_GNU}/src/nproc||command -v nproc) +MAKEFLAGS="${MAKEFLAGS} -j ${NPROC}" export MAKEFLAGS ### From 242bfb45458d7c5910f0bd6877848d7ce0d5395e Mon Sep 17 00:00:00 2001 From: oech3 <79379754+oech3@users.noreply.github.com> Date: Sat, 20 Dec 2025 19:07:40 +0900 Subject: [PATCH 4/7] why-error.md: Cleanup (#9738) --- util/why-error.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/util/why-error.md b/util/why-error.md index 137e189ad81..04039e34e2a 100644 --- a/util/why-error.md +++ b/util/why-error.md @@ -13,18 +13,15 @@ This file documents why some GNU tests are failing: * ls/ls-misc.pl * ls/stat-free-symlinks.sh * misc/close-stdout.sh -* misc/nohup.sh * numfmt/numfmt.pl - https://github.com/uutils/coreutils/issues/7219 / https://github.com/uutils/coreutils/issues/7221 * misc/stdbuf.sh - https://github.com/uutils/coreutils/issues/7072 * misc/tsort.pl - https://github.com/uutils/coreutils/issues/7074 * misc/write-errors.sh -* od/od-float.sh * ptx/ptx-overrun.sh * ptx/ptx.pl * rm/one-file-system.sh - https://github.com/uutils/coreutils/issues/7011 * rm/rm1.sh - https://github.com/uutils/coreutils/issues/9479 -* shred/shred-passes.sh -* sort/sort-continue.sh +* shred/shred-passes.sh - https://github.com/uutils/coreutils/pull/9317 * sort/sort-debug-keys.sh * sort/sort-debug-warn.sh * sort/sort-float.sh @@ -39,4 +36,3 @@ This file documents why some GNU tests are failing: * tail/symlink.sh * stty/stty-row-col.sh * stty/stty.sh -* tty/tty-eof.pl From d6140576a684aab7768f1c6a90cf7923f9470d9c Mon Sep 17 00:00:00 2001 From: oech3 <79379754+oech3@users.noreply.github.com> Date: Sat, 20 Dec 2025 19:10:43 +0900 Subject: [PATCH 5/7] build-gnu.sh: Move {ch,run}con tests to SELinux VM to avoid wrong result by false symlinks (#9607) --- util/build-gnu.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/util/build-gnu.sh b/util/build-gnu.sh index 6d5f622d137..25ff4cc6a23 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -105,7 +105,8 @@ test -f "${UU_BUILD_DIR}/[" || (cd ${UU_BUILD_DIR} && ln -s "test" "[") cd "${path_GNU}" && echo "[ pwd:'${PWD}' ]" -# Any binaries that aren't built become `false` so their tests fail +# Any binaries that aren't built become `false` to make tests failure +# Note that some test (e.g. runcon/runcon-compute.sh) incorrectly passes by this for binary in $(./build-aux/gen-lists-of-programs.sh --list-progs); do bin_path="${UU_BUILD_DIR}/${binary}" test -f "${bin_path}" || { @@ -166,6 +167,11 @@ grep -rl 'path_prepend_' tests/* | xargs -r "${SED}" -i 's| path_prepend_ ./src| # path_prepend_ sets $abs_path_dir_: set it manually instead. grep -rl '\$abs_path_dir_' tests/*/*.sh | xargs -r "${SED}" -i "s|\$abs_path_dir_|${UU_BUILD_DIR//\//\\/}|g" +# We can't build runcon and chcon without libselinux. But GNU no longer builds dummies of them. So consider they are SELinux specific. +"${SED}" -i 's/^print_ver_.*/require_selinux_/' tests/runcon/runcon-compute.sh +"${SED}" -i 's/^print_ver_.*/require_selinux_/' tests/runcon/runcon-no-reorder.sh +"${SED}" -i 's/^print_ver_.*/require_selinux_/' tests/chcon/chcon-fail.sh + # We use coreutils yes "${SED}" -i "s|--coreutils-prog=||g" tests/misc/coreutils.sh # Different message From c0f0fc77da3fcd78845e5d9139c456d4819d10f1 Mon Sep 17 00:00:00 2001 From: oech3 <79379754+oech3@users.noreply.github.com> Date: Sat, 20 Dec 2025 20:43:00 +0900 Subject: [PATCH 6/7] build-gnu.sh: Remove a hack to force-enable tests & some fix --- util/build-gnu.sh | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/util/build-gnu.sh b/util/build-gnu.sh index 25ff4cc6a23..f18e577db01 100755 --- a/util/build-gnu.sh +++ b/util/build-gnu.sh @@ -5,7 +5,6 @@ # spell-checker:ignore (paths) abmon deref discrim eacces getlimits getopt ginstall inacc infloop inotify reflink ; (misc) INT_OFLOW OFLOW # spell-checker:ignore baddecode submodules xstrtol distros ; (vars/env) SRCDIR vdir rcexp xpart dired OSTYPE ; (utils) greadlink gsed multihardlink texinfo CARGOFLAGS # spell-checker:ignore openat TOCTOU CFLAGS -# spell-checker:ignore hfsplus casefold chattr set -e @@ -128,7 +127,7 @@ else "${SED}" -i 's|check-texinfo: $(syntax_checks)|check-texinfo:|' doc/local.mk # Use CFLAGS for best build time since we discard GNU coreutils CFLAGS="${CFLAGS} -pipe -O0 -s" ./configure -C --quiet --disable-gcc-warnings --disable-nls --disable-dependency-tracking --disable-bold-man-page-references \ - --enable-single-binary=symlinks \ + --enable-single-binary=symlinks --enable-install-program="arch,kill,uptime,hostname" \ "$([ "${SELINUX_ENABLED}" = 1 ] && echo --with-selinux || echo --without-selinux)" #Add timeout to to protect against hangs "${SED}" -i 's|^"\$@|'"${SYSTEM_TIMEOUT}"' 600 "\$@|' build-aux/test-driver @@ -168,7 +167,7 @@ grep -rl 'path_prepend_' tests/* | xargs -r "${SED}" -i 's| path_prepend_ ./src| grep -rl '\$abs_path_dir_' tests/*/*.sh | xargs -r "${SED}" -i "s|\$abs_path_dir_|${UU_BUILD_DIR//\//\\/}|g" # We can't build runcon and chcon without libselinux. But GNU no longer builds dummies of them. So consider they are SELinux specific. -"${SED}" -i 's/^print_ver_.*/require_selinux_/' tests/runcon/runcon-compute.sh +#"${SED}" -i 's/^print_ver_.*/require_selinux_/' tests/runcon/runcon-compute.sh #this test incorrectly passes with false symlink "${SED}" -i 's/^print_ver_.*/require_selinux_/' tests/runcon/runcon-no-reorder.sh "${SED}" -i 's/^print_ver_.*/require_selinux_/' tests/chcon/chcon-fail.sh @@ -249,9 +248,6 @@ sed -i -e "s|---dis ||g" tests/tail/overlay-headers.sh "${SED}" -i "s/ {ERR=>\"\$prog: foobar\\\\n\" \. \$try_help }/ {ERR=>\"error: unexpected argument '--foobar' found\n\n tip: to pass '--foobar' as a value, use '-- --foobar'\n\nUsage: basenc [OPTION]... [FILE]\n\nFor more information, try '--help'.\n\"}]/" tests/basenc/basenc.pl "${SED}" -i "s/ {ERR_SUBST=>\"s\/(unrecognized|unknown) option \[-' \]\*foobar\[' \]\*\/foobar\/\"}],//" tests/basenc/basenc.pl -# Remove the check whether a util was built. Otherwise tests against utils like "arch" are not run. -"${SED}" -i "s|require_built_ |# require_built_ |g" init.cfg - # exit early for the selinux check. The first is enough for us. "${SED}" -i "s|# Independent of whether SELinux|return 0\n #|g" init.cfg From 2889f0cfda453ef360ea48962b99760e10afc4a8 Mon Sep 17 00:00:00 2001 From: oech3 <79379754+oech3@users.noreply.github.com> Date: Sat, 20 Dec 2025 19:19:34 +0900 Subject: [PATCH 7/7] GnuTests.yml: Fix caches --- .github/workflows/GnuTests.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/GnuTests.yml b/.github/workflows/GnuTests.yml index 6c528dbd32a..3d0477fbb55 100644 --- a/.github/workflows/GnuTests.yml +++ b/.github/workflows/GnuTests.yml @@ -58,10 +58,8 @@ jobs: path: | gnu/config.cache gnu/src/getlimits - key: ${{ runner.os }}-gnu-config-${{ env.REPO_GNU_REF }}-${{ hashFiles('gnu/configure') }} - restore-keys: | - ${{ runner.os }}-gnu-config-${{ env.REPO_GNU_REF }}- - ${{ runner.os }}-gnu-config- + key: ${{ runner.os }}-gnu-config-${{ hashFiles('gnu/NEWS') }}-${{ hashFiles('gnu/configure') }} + #### Build environment setup - name: Install dependencies shell: bash @@ -112,7 +110,7 @@ jobs: path: | gnu/config.cache gnu/src/getlimits - key: ${{ runner.os }}-gnu-config-${{ env.REPO_GNU_REF }}-${{ hashFiles('gnu/configure') }} + key: ${{ runner.os }}-gnu-config-${{ hashFiles('gnu/NEWS') }}-${{ hashFiles('gnu/configure') }} ### Run tests as user - name: Run GNU tests