Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions .github/workflows/GnuTests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -58,10 +58,8 @@ jobs:
path: |
gnu/config.cache
gnu/src/getlimits
key: ${{ runner.os }}-gnu-config-${{ env.REPO_GNU_REF }}-${{ hashFiles('gnu/configure') }}
restore-keys: |
${{ runner.os }}-gnu-config-${{ env.REPO_GNU_REF }}-
${{ runner.os }}-gnu-config-
key: ${{ runner.os }}-gnu-config-${{ hashFiles('gnu/NEWS') }}-${{ hashFiles('gnu/configure') }}

#### Build environment setup
- name: Install dependencies
shell: bash
Expand Down Expand Up @@ -112,7 +110,7 @@ jobs:
path: |
gnu/config.cache
gnu/src/getlimits
key: ${{ runner.os }}-gnu-config-${{ env.REPO_GNU_REF }}-${{ hashFiles('gnu/configure') }}
key: ${{ runner.os }}-gnu-config-${{ hashFiles('gnu/NEWS') }}-${{ hashFiles('gnu/configure') }}

### Run tests as user
- name: Run GNU tests
Expand Down
4 changes: 2 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

39 changes: 32 additions & 7 deletions src/uu/sort/src/merge.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ use uucore::error::{FromIo, UResult};
use crate::{
GlobalSettings, Output, SortError,
chunks::{self, Chunk, RecycledChunk},
compare_by, open,
compare_by, fd_soft_limit, open,
tmp_dir::TmpDirWrapper,
};

Expand Down Expand Up @@ -62,6 +62,28 @@ fn replace_output_file_in_input_files(
Ok(())
}

/// Determine the effective merge batch size, enforcing a minimum and respecting the
/// file-descriptor soft limit after reserving stdio/output and a safety margin.
fn effective_merge_batch_size(settings: &GlobalSettings) -> usize {
const MIN_BATCH_SIZE: usize = 2;
const RESERVED_STDIO: usize = 3;
const RESERVED_OUTPUT: usize = 1;
const SAFETY_MARGIN: usize = 1;
let mut batch_size = settings.merge_batch_size.max(MIN_BATCH_SIZE);

if let Some(limit) = fd_soft_limit() {
let reserved = RESERVED_STDIO + RESERVED_OUTPUT + SAFETY_MARGIN;
let available_inputs = limit.saturating_sub(reserved);
if available_inputs >= MIN_BATCH_SIZE {
batch_size = batch_size.min(available_inputs);
} else {
batch_size = MIN_BATCH_SIZE;
}
}

batch_size
}

/// Merge pre-sorted `Box<dyn Read>`s.
///
/// If `settings.merge_batch_size` is greater than the length of `files`, intermediate files will be used.
Expand Down Expand Up @@ -94,18 +116,21 @@ pub fn merge_with_file_limit<
output: Output,
tmp_dir: &mut TmpDirWrapper,
) -> UResult<()> {
if files.len() <= settings.merge_batch_size {
let batch_size = effective_merge_batch_size(settings);
debug_assert!(batch_size >= 2);

if files.len() <= batch_size {
let merger = merge_without_limit(files, settings);
merger?.write_all(settings, output)
} else {
let mut temporary_files = vec![];
let mut batch = vec![];
let mut batch = Vec::with_capacity(batch_size);
for file in files {
batch.push(file);
if batch.len() >= settings.merge_batch_size {
assert_eq!(batch.len(), settings.merge_batch_size);
if batch.len() >= batch_size {
assert_eq!(batch.len(), batch_size);
let merger = merge_without_limit(batch.into_iter(), settings)?;
batch = vec![];
batch = Vec::with_capacity(batch_size);

let mut tmp_file =
Tmp::create(tmp_dir.next_file()?, settings.compress_prog.as_deref())?;
Expand All @@ -115,7 +140,7 @@ pub fn merge_with_file_limit<
}
// Merge any remaining files that didn't get merged in a full batch above.
if !batch.is_empty() {
assert!(batch.len() < settings.merge_batch_size);
assert!(batch.len() < batch_size);
let merger = merge_without_limit(batch.into_iter(), settings)?;

let mut tmp_file =
Expand Down
48 changes: 38 additions & 10 deletions src/uu/sort/src/sort.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1073,13 +1073,27 @@ fn make_sort_mode_arg(mode: &'static str, short: char, help: String) -> Arg {

#[cfg(target_os = "linux")]
fn get_rlimit() -> UResult<usize> {
use nix::sys::resource::{Resource, getrlimit};
use nix::sys::resource::{RLIM_INFINITY, Resource, getrlimit};

getrlimit(Resource::RLIMIT_NOFILE)
.map(|(rlim_cur, _)| rlim_cur as usize)
let (rlim_cur, _rlim_max) = getrlimit(Resource::RLIMIT_NOFILE)
.map_err(|_| UUsageError::new(2, translate!("sort-failed-fetch-rlimit")))?;
if rlim_cur == RLIM_INFINITY {
return Err(UUsageError::new(2, translate!("sort-failed-fetch-rlimit")));
}
usize::try_from(rlim_cur)
.map_err(|_| UUsageError::new(2, translate!("sort-failed-fetch-rlimit")))
}

#[cfg(target_os = "linux")]
pub(crate) fn fd_soft_limit() -> Option<usize> {
get_rlimit().ok()
}

#[cfg(not(target_os = "linux"))]
pub(crate) fn fd_soft_limit() -> Option<usize> {
None
}

const STDIN_FILE: &str = "-";

/// Legacy `+POS1 [-POS2]` syntax is permitted unless `_POSIX2_VERSION` is in
Expand Down Expand Up @@ -1232,12 +1246,12 @@ fn default_merge_batch_size() -> usize {
#[cfg(target_os = "linux")]
{
// Adjust merge batch size dynamically based on available file descriptors.
match get_rlimit() {
Ok(limit) => {
match fd_soft_limit() {
Some(limit) => {
let usable_limit = limit.saturating_div(LINUX_BATCH_DIVISOR);
usable_limit.clamp(LINUX_BATCH_MIN, LINUX_BATCH_MAX)
}
Err(_) => 64,
None => 64,
}
}

Expand Down Expand Up @@ -1366,9 +1380,15 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {
settings.threads = matches
.get_one::<String>(options::PARALLEL)
.map_or_else(|| "0".to_string(), String::from);
unsafe {
env::set_var("RAYON_NUM_THREADS", &settings.threads);
}
let num_threads = match settings.threads.parse::<usize>() {
Ok(0) | Err(_) => std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1),
Ok(n) => n,
};
let _ = rayon::ThreadPoolBuilder::new()
.num_threads(num_threads)
.build_global();
}

if let Some(size_str) = matches.get_one::<String>(options::BUF_SIZE) {
Expand Down Expand Up @@ -1419,7 +1439,15 @@ pub fn uumain(args: impl uucore::Args) -> UResult<()> {

translate!(
"sort-maximum-batch-size-rlimit",
"rlimit" => get_rlimit()?
"rlimit" => {
let Some(rlimit) = fd_soft_limit() else {
return Err(UUsageError::new(
2,
translate!("sort-failed-fetch-rlimit"),
));
};
rlimit
}
)
}
#[cfg(not(target_os = "linux"))]
Expand Down
14 changes: 8 additions & 6 deletions util/build-gnu.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
# spell-checker:ignore (paths) abmon deref discrim eacces getlimits getopt ginstall inacc infloop inotify reflink ; (misc) INT_OFLOW OFLOW
# spell-checker:ignore baddecode submodules xstrtol distros ; (vars/env) SRCDIR vdir rcexp xpart dired OSTYPE ; (utils) greadlink gsed multihardlink texinfo CARGOFLAGS
# spell-checker:ignore openat TOCTOU CFLAGS
# spell-checker:ignore hfsplus casefold chattr

set -e

Expand Down Expand Up @@ -105,7 +104,8 @@ test -f "${UU_BUILD_DIR}/[" || (cd ${UU_BUILD_DIR} && ln -s "test" "[")

cd "${path_GNU}" && echo "[ pwd:'${PWD}' ]"

# Any binaries that aren't built become `false` so their tests fail
# Any binaries that aren't built become `false` to make tests failure
# Note that some test (e.g. runcon/runcon-compute.sh) incorrectly passes by this
for binary in $(./build-aux/gen-lists-of-programs.sh --list-progs); do
bin_path="${UU_BUILD_DIR}/${binary}"
test -f "${bin_path}" || {
Expand All @@ -127,7 +127,7 @@ else
"${SED}" -i 's|check-texinfo: $(syntax_checks)|check-texinfo:|' doc/local.mk
# Use CFLAGS for best build time since we discard GNU coreutils
CFLAGS="${CFLAGS} -pipe -O0 -s" ./configure -C --quiet --disable-gcc-warnings --disable-nls --disable-dependency-tracking --disable-bold-man-page-references \
--enable-single-binary=symlinks \
--enable-single-binary=symlinks --enable-install-program="arch,kill,uptime,hostname" \
"$([ "${SELINUX_ENABLED}" = 1 ] && echo --with-selinux || echo --without-selinux)"
#Add timeout to to protect against hangs
"${SED}" -i 's|^"\$@|'"${SYSTEM_TIMEOUT}"' 600 "\$@|' build-aux/test-driver
Expand Down Expand Up @@ -166,6 +166,11 @@ grep -rl 'path_prepend_' tests/* | xargs -r "${SED}" -i 's| path_prepend_ ./src|
# path_prepend_ sets $abs_path_dir_: set it manually instead.
grep -rl '\$abs_path_dir_' tests/*/*.sh | xargs -r "${SED}" -i "s|\$abs_path_dir_|${UU_BUILD_DIR//\//\\/}|g"

# We can't build runcon and chcon without libselinux. But GNU no longer builds dummies of them. So consider they are SELinux specific.
#"${SED}" -i 's/^print_ver_.*/require_selinux_/' tests/runcon/runcon-compute.sh #this test incorrectly passes with false symlink
"${SED}" -i 's/^print_ver_.*/require_selinux_/' tests/runcon/runcon-no-reorder.sh
"${SED}" -i 's/^print_ver_.*/require_selinux_/' tests/chcon/chcon-fail.sh

# We use coreutils yes
"${SED}" -i "s|--coreutils-prog=||g" tests/misc/coreutils.sh
# Different message
Expand Down Expand Up @@ -243,9 +248,6 @@ sed -i -e "s|---dis ||g" tests/tail/overlay-headers.sh
"${SED}" -i "s/ {ERR=>\"\$prog: foobar\\\\n\" \. \$try_help }/ {ERR=>\"error: unexpected argument '--foobar' found\n\n tip: to pass '--foobar' as a value, use '-- --foobar'\n\nUsage: basenc [OPTION]... [FILE]\n\nFor more information, try '--help'.\n\"}]/" tests/basenc/basenc.pl
"${SED}" -i "s/ {ERR_SUBST=>\"s\/(unrecognized|unknown) option \[-' \]\*foobar\[' \]\*\/foobar\/\"}],//" tests/basenc/basenc.pl

# Remove the check whether a util was built. Otherwise tests against utils like "arch" are not run.
"${SED}" -i "s|require_built_ |# require_built_ |g" init.cfg

# exit early for the selinux check. The first is enough for us.
"${SED}" -i "s|# Independent of whether SELinux|return 0\n #|g" init.cfg

Expand Down
3 changes: 2 additions & 1 deletion util/run-gnu-test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ echo "path_UUTILS='${path_UUTILS}'"
echo "path_GNU='${path_GNU}'"

# Use GNU nproc for *BSD
MAKEFLAGS="${MAKEFLAGS} -j $(${path_GNU}/src/nproc)"
NPROC=$(command -v ${path_GNU}/src/nproc||command -v nproc)
MAKEFLAGS="${MAKEFLAGS} -j ${NPROC}"
export MAKEFLAGS
###

Expand Down
6 changes: 1 addition & 5 deletions util/why-error.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,15 @@ This file documents why some GNU tests are failing:
* ls/ls-misc.pl
* ls/stat-free-symlinks.sh
* misc/close-stdout.sh
* misc/nohup.sh
* numfmt/numfmt.pl - https://github.com/uutils/coreutils/issues/7219 / https://github.com/uutils/coreutils/issues/7221
* misc/stdbuf.sh - https://github.com/uutils/coreutils/issues/7072
* misc/tsort.pl - https://github.com/uutils/coreutils/issues/7074
* misc/write-errors.sh
* od/od-float.sh
* ptx/ptx-overrun.sh
* ptx/ptx.pl
* rm/one-file-system.sh - https://github.com/uutils/coreutils/issues/7011
* rm/rm1.sh - https://github.com/uutils/coreutils/issues/9479
* shred/shred-passes.sh
* sort/sort-continue.sh
* shred/shred-passes.sh - https://github.com/uutils/coreutils/pull/9317
* sort/sort-debug-keys.sh
* sort/sort-debug-warn.sh
* sort/sort-float.sh
Expand All @@ -39,4 +36,3 @@ This file documents why some GNU tests are failing:
* tail/symlink.sh
* stty/stty-row-col.sh
* stty/stty.sh
* tty/tty-eof.pl