From 6d721e5d1efefb7ee18ab7694687e44145ee8885 Mon Sep 17 00:00:00 2001 From: mattsu Date: Fri, 14 Nov 2025 20:30:56 +0900 Subject: [PATCH 01/15] feat(fold): add column counting for character width mode in process_ascii_line Implement logic to increment column count in WidthMode::Characters, emitting output when width is reached. This ensures accurate line folding for multi-byte characters, enhancing Unicode support. --- src/uu/fold/src/fold.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/uu/fold/src/fold.rs b/src/uu/fold/src/fold.rs index f14ed3cf071..c13ac93f399 100644 --- a/src/uu/fold/src/fold.rs +++ b/src/uu/fold/src/fold.rs @@ -368,6 +368,13 @@ fn process_ascii_line(line: &[u8], ctx: &mut FoldContext<'_, W>) -> UR } else if !ctx.spaces { *ctx.last_space = None; } + + if ctx.mode == WidthMode::Characters { + *ctx.col_count = ctx.col_count.saturating_add(1); + if *ctx.col_count >= ctx.width { + emit_output(ctx)?; + } + } idx += 1; } _ => { From b21cf3549d2791703f96ac032ef1e42196e524d8 Mon Sep 17 00:00:00 2001 From: mattsu Date: Fri, 14 Nov 2025 23:18:53 +0900 Subject: [PATCH 02/15] fix fold: emit output early when column count reaches width limit - Added conditional check in fold_file function to call emit_output when col_count >= width - Ensures lines are properly wrapped based on byte or character width before final output flush - Improves handling of incomplete lines that need early breaking to respect the specified width --- src/uu/fold/src/fold.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/src/uu/fold/src/fold.rs b/src/uu/fold/src/fold.rs index c13ac93f399..e42c0f02048 100644 --- a/src/uu/fold/src/fold.rs +++ b/src/uu/fold/src/fold.rs @@ -603,8 +603,23 @@ fn fold_file( } if !output.is_empty() { - writer.write_all(&output)?; - output.clear(); + if col_count >= width { + let mut ctx = FoldContext { + spaces, + width, + mode, + writer, + output: &mut output, + col_count: &mut col_count, + last_space: &mut last_space, + }; + emit_output(&mut ctx)?; + } + + if !output.is_empty() { + writer.write_all(&output)?; + output.clear(); + } } Ok(()) From 77a1c31c43817cb235b974e9b909e150679925e8 Mon Sep 17 00:00:00 2001 From: mattsu Date: Fri, 14 Nov 2025 23:35:17 +0900 Subject: [PATCH 03/15] fix: correct output emission logic in fold for character mode In character width mode, emit output immediately after segments are added if column count exceeds width, preventing redundant flushes. Simplify the file folding logic by removing unnecessary conditional checks at the end, ensuring clean output writing. This fixes potential issues with extra line breaks or incorrect folding behavior. --- src/uu/fold/src/fold.rs | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/src/uu/fold/src/fold.rs b/src/uu/fold/src/fold.rs index e42c0f02048..6a52cedc4ca 100644 --- a/src/uu/fold/src/fold.rs +++ b/src/uu/fold/src/fold.rs @@ -427,6 +427,11 @@ fn push_ascii_segment(segment: &[u8], ctx: &mut FoldContext<'_, W>) -> } remaining = &remaining[take..]; + + if ctx.mode == WidthMode::Characters && *ctx.col_count >= ctx.width && !ctx.output.is_empty() + { + emit_output(ctx)?; + } } Ok(()) @@ -507,6 +512,11 @@ fn process_utf8_line(line: &str, ctx: &mut FoldContext<'_, W>) -> URes ctx.output .extend_from_slice(&line_bytes[byte_idx..next_idx]); *ctx.col_count = ctx.col_count.saturating_add(added); + + if ctx.mode == WidthMode::Characters && *ctx.col_count >= ctx.width && !ctx.output.is_empty() + { + emit_output(ctx)?; + } } Ok(()) @@ -549,6 +559,11 @@ fn process_non_utf8_line(line: &[u8], ctx: &mut FoldContext<'_, W>) -> } ctx.output.push(byte); + + if ctx.mode == WidthMode::Characters && *ctx.col_count >= ctx.width && !ctx.output.is_empty() + { + emit_output(ctx)?; + } } Ok(()) @@ -603,23 +618,8 @@ fn fold_file( } if !output.is_empty() { - if col_count >= width { - let mut ctx = FoldContext { - spaces, - width, - mode, - writer, - output: &mut output, - col_count: &mut col_count, - last_space: &mut last_space, - }; - emit_output(&mut ctx)?; - } - - if !output.is_empty() { - writer.write_all(&output)?; - output.clear(); - } + writer.write_all(&output)?; + output.clear(); } Ok(()) From b0e0033a5812ba1cc0dcc47e4ff92e63eedfebf1 Mon Sep 17 00:00:00 2001 From: mattsu Date: Fri, 14 Nov 2025 23:36:19 +0900 Subject: [PATCH 04/15] refactor(fold): split long if-conditions into multiple lines for readability Refactor code in fold.rs to break lengthy if-condition statements across multiple lines in push_ascii_segment, process_utf8_line, and process_non_utf8_line functions. This improves code readability without changing functionality. --- src/uu/fold/src/fold.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/uu/fold/src/fold.rs b/src/uu/fold/src/fold.rs index 6a52cedc4ca..9e7381ddbdd 100644 --- a/src/uu/fold/src/fold.rs +++ b/src/uu/fold/src/fold.rs @@ -428,7 +428,9 @@ fn push_ascii_segment(segment: &[u8], ctx: &mut FoldContext<'_, W>) -> remaining = &remaining[take..]; - if ctx.mode == WidthMode::Characters && *ctx.col_count >= ctx.width && !ctx.output.is_empty() + if ctx.mode == WidthMode::Characters + && *ctx.col_count >= ctx.width + && !ctx.output.is_empty() { emit_output(ctx)?; } @@ -513,7 +515,9 @@ fn process_utf8_line(line: &str, ctx: &mut FoldContext<'_, W>) -> URes .extend_from_slice(&line_bytes[byte_idx..next_idx]); *ctx.col_count = ctx.col_count.saturating_add(added); - if ctx.mode == WidthMode::Characters && *ctx.col_count >= ctx.width && !ctx.output.is_empty() + if ctx.mode == WidthMode::Characters + && *ctx.col_count >= ctx.width + && !ctx.output.is_empty() { emit_output(ctx)?; } @@ -560,7 +564,9 @@ fn process_non_utf8_line(line: &[u8], ctx: &mut FoldContext<'_, W>) -> ctx.output.push(byte); - if ctx.mode == WidthMode::Characters && *ctx.col_count >= ctx.width && !ctx.output.is_empty() + if ctx.mode == WidthMode::Characters + && *ctx.col_count >= ctx.width + && !ctx.output.is_empty() { emit_output(ctx)?; } From ac09d10f0d06e711befa7eeae6fd2949abe8ca76 Mon Sep 17 00:00:00 2001 From: mattsu Date: Sat, 15 Nov 2025 00:10:39 +0900 Subject: [PATCH 05/15] feat(fold): add streaming output with periodic flushing to reduce memory usage Introduce a STREAMING_FLUSH_THRESHOLD constant and helper functions (maybe_flush_unbroken_output, push_byte, push_bytes) to periodically flush the output buffer when it exceeds 8KB and no spaces are being tracked, preventing excessive memory consumption when processing large files. This refactor replaces direct buffer pushes with checks for threshold-based flushing. --- src/uu/fold/src/fold.rs | 52 +++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/src/uu/fold/src/fold.rs b/src/uu/fold/src/fold.rs index 9e7381ddbdd..4f99fffa76b 100644 --- a/src/uu/fold/src/fold.rs +++ b/src/uu/fold/src/fold.rs @@ -19,6 +19,7 @@ const TAB_WIDTH: usize = 8; const NL: u8 = b'\n'; const CR: u8 = b'\r'; const TAB: u8 = b'\t'; +const STREAMING_FLUSH_THRESHOLD: usize = 8 * 1024; mod options { pub const BYTES: &str = "bytes"; @@ -322,6 +323,31 @@ fn emit_output(ctx: &mut FoldContext<'_, W>) -> UResult<()> { Ok(()) } +fn maybe_flush_unbroken_output(ctx: &mut FoldContext<'_, W>) -> UResult<()> { + if ctx.spaces || ctx.output.len() < STREAMING_FLUSH_THRESHOLD { + return Ok(()); + } + + if !ctx.output.is_empty() { + ctx.writer.write_all(ctx.output)?; + ctx.output.clear(); + } + Ok(()) +} + +fn push_byte(ctx: &mut FoldContext<'_, W>, byte: u8) -> UResult<()> { + ctx.output.push(byte); + maybe_flush_unbroken_output(ctx) +} + +fn push_bytes(ctx: &mut FoldContext<'_, W>, bytes: &[u8]) -> UResult<()> { + if bytes.is_empty() { + return Ok(()); + } + ctx.output.extend_from_slice(bytes); + maybe_flush_unbroken_output(ctx) +} + fn process_ascii_line(line: &[u8], ctx: &mut FoldContext<'_, W>) -> UResult<()> { let mut idx = 0; let len = line.len(); @@ -334,12 +360,12 @@ fn process_ascii_line(line: &[u8], ctx: &mut FoldContext<'_, W>) -> UR break; } CR => { - ctx.output.push(CR); + push_byte(ctx, CR)?; *ctx.col_count = 0; idx += 1; } 0x08 => { - ctx.output.push(0x08); + push_byte(ctx, 0x08)?; *ctx.col_count = ctx.col_count.saturating_sub(1); idx += 1; } @@ -358,11 +384,11 @@ fn process_ascii_line(line: &[u8], ctx: &mut FoldContext<'_, W>) -> UR } else { *ctx.last_space = None; } - ctx.output.push(TAB); + push_byte(ctx, TAB)?; idx += 1; } 0x00..=0x07 | 0x0B..=0x0C | 0x0E..=0x1F | 0x7F => { - ctx.output.push(line[idx]); + push_byte(ctx, line[idx])?; if ctx.spaces && line[idx].is_ascii_whitespace() && line[idx] != CR { *ctx.last_space = Some(ctx.output.len() - 1); } else if !ctx.spaces { @@ -412,7 +438,7 @@ fn push_ascii_segment(segment: &[u8], ctx: &mut FoldContext<'_, W>) -> let take = remaining.len().min(available); let base_len = ctx.output.len(); - ctx.output.extend_from_slice(&remaining[..take]); + push_bytes(ctx, &remaining[..take])?; *ctx.col_count += take; if ctx.spaces { @@ -461,15 +487,13 @@ fn process_utf8_line(line: &str, ctx: &mut FoldContext<'_, W>) -> URes } if ch == '\r' { - ctx.output - .extend_from_slice(&line_bytes[byte_idx..next_idx]); + push_bytes(ctx, &line_bytes[byte_idx..next_idx])?; *ctx.col_count = 0; continue; } if ch == '\x08' { - ctx.output - .extend_from_slice(&line_bytes[byte_idx..next_idx]); + push_bytes(ctx, &line_bytes[byte_idx..next_idx])?; *ctx.col_count = ctx.col_count.saturating_sub(1); continue; } @@ -489,8 +513,7 @@ fn process_utf8_line(line: &str, ctx: &mut FoldContext<'_, W>) -> URes } else { *ctx.last_space = None; } - ctx.output - .extend_from_slice(&line_bytes[byte_idx..next_idx]); + push_bytes(ctx, &line_bytes[byte_idx..next_idx])?; continue; } @@ -511,8 +534,7 @@ fn process_utf8_line(line: &str, ctx: &mut FoldContext<'_, W>) -> URes *ctx.last_space = Some(ctx.output.len()); } - ctx.output - .extend_from_slice(&line_bytes[byte_idx..next_idx]); + push_bytes(ctx, &line_bytes[byte_idx..next_idx])?; *ctx.col_count = ctx.col_count.saturating_add(added); if ctx.mode == WidthMode::Characters @@ -551,7 +573,7 @@ fn process_non_utf8_line(line: &[u8], ctx: &mut FoldContext<'_, W>) -> } else { None }; - ctx.output.push(byte); + push_byte(ctx, byte)?; continue; } 0x08 => *ctx.col_count = ctx.col_count.saturating_sub(1), @@ -562,7 +584,7 @@ fn process_non_utf8_line(line: &[u8], ctx: &mut FoldContext<'_, W>) -> _ => *ctx.col_count = ctx.col_count.saturating_add(1), } - ctx.output.push(byte); + push_byte(ctx, byte)?; if ctx.mode == WidthMode::Characters && *ctx.col_count >= ctx.width From dba5b9bb38a043dd60a479dbf34cd670ae26a582 Mon Sep 17 00:00:00 2001 From: mattsu Date: Sat, 15 Nov 2025 08:55:52 +0900 Subject: [PATCH 06/15] fix(fold): correct premature output emission in character mode and add tests Remove conditional checks that incorrectly emitted output when column count reached width in character mode, ensuring proper folding of wide characters and handling of edge cases. Add comprehensive tests for wide characters, invalid UTF-8, zero-width spaces, and buffer boundaries to verify correct behavior. This prevents issues with multi-byte character folding where output was prematurely flushed, improving accuracy for Unicode input. --- src/uu/fold/src/fold.rs | 19 ---- tests/by-util/test_fold.rs | 215 +++++++++++++++++++++++++++++++++++++ 2 files changed, 215 insertions(+), 19 deletions(-) diff --git a/src/uu/fold/src/fold.rs b/src/uu/fold/src/fold.rs index 4f99fffa76b..2ebfa65c248 100644 --- a/src/uu/fold/src/fold.rs +++ b/src/uu/fold/src/fold.rs @@ -454,12 +454,6 @@ fn push_ascii_segment(segment: &[u8], ctx: &mut FoldContext<'_, W>) -> remaining = &remaining[take..]; - if ctx.mode == WidthMode::Characters - && *ctx.col_count >= ctx.width - && !ctx.output.is_empty() - { - emit_output(ctx)?; - } } Ok(()) @@ -537,12 +531,6 @@ fn process_utf8_line(line: &str, ctx: &mut FoldContext<'_, W>) -> URes push_bytes(ctx, &line_bytes[byte_idx..next_idx])?; *ctx.col_count = ctx.col_count.saturating_add(added); - if ctx.mode == WidthMode::Characters - && *ctx.col_count >= ctx.width - && !ctx.output.is_empty() - { - emit_output(ctx)?; - } } Ok(()) @@ -585,13 +573,6 @@ fn process_non_utf8_line(line: &[u8], ctx: &mut FoldContext<'_, W>) -> } push_byte(ctx, byte)?; - - if ctx.mode == WidthMode::Characters - && *ctx.col_count >= ctx.width - && !ctx.output.is_empty() - { - emit_output(ctx)?; - } } Ok(()) diff --git a/tests/by-util/test_fold.rs b/tests/by-util/test_fold.rs index 04072ab157f..f15199fb612 100644 --- a/tests/by-util/test_fold.rs +++ b/tests/by-util/test_fold.rs @@ -59,6 +59,221 @@ fn test_wide_characters_with_characters_option() { .stdout_is("\u{B250}\u{B250}\u{B250}\n"); } +#[test] +fn test_multiple_wide_characters_in_column_mode() { + let wide = '\u{FF1A}'; + let mut input = wide.to_string().repeat(50); + input.push('\n'); + + let mut expected = String::new(); + for i in 1..=50 { + expected.push(wide); + if i % 5 == 0 { + expected.push('\n'); + } + } + + new_ucmd!() + .args(&["-w", "10"]) + .pipe_in(input) + .succeeds() + .stdout_is(expected); +} + +#[test] +fn test_multiple_wide_characters_in_character_mode() { + let wide = '\u{FF1A}'; + let mut input = wide.to_string().repeat(50); + input.push('\n'); + + let mut expected = String::new(); + for i in 1..=50 { + expected.push(wide); + if i % 10 == 0 { + expected.push('\n'); + } + } + + new_ucmd!() + .args(&["--characters", "-w", "10"]) + .pipe_in(input) + .succeeds() + .stdout_is(expected); +} + +#[test] +fn test_unicode_on_reader_buffer_boundary_in_character_mode() { + let boundary = buf_reader_capacity().saturating_sub(1); + assert!(boundary > 0, "BufReader capacity must be greater than 1"); + + let mut input = "a".repeat(boundary); + input.push('\u{B250}'); + input.push_str(&"a".repeat(100)); + input.push('\n'); + + let expected_tail = tail_inclusive( + &fold_characters_reference(&input, 80), + 4, + ); + + let result = new_ucmd!() + .arg("--characters") + .pipe_in(input) + .succeeds(); + + let actual_tail = tail_inclusive(result.stdout_str(), 4); + + assert_eq!(actual_tail, expected_tail); +} + +#[test] +fn test_fold_preserves_invalid_utf8_sequences() { + let bad_input: &[u8] = b"\xC3|\xED\xBA\xAD|\x00|\x89|\xED\xA6\xBF\xED\xBF\xBF\n"; + + new_ucmd!() + .pipe_in(bad_input.to_vec()) + .succeeds() + .stdout_is_bytes(bad_input); +} + +#[test] +fn test_fold_preserves_incomplete_utf8_at_eof() { + let trailing_byte: &[u8] = b"\xC3"; + + new_ucmd!() + .pipe_in(trailing_byte.to_vec()) + .succeeds() + .stdout_is_bytes(trailing_byte); +} + +#[test] +fn test_zero_width_bytes_in_column_mode() { + let len = io_buf_size_times_two(); + let input = vec![0u8; len]; + + new_ucmd!() + .pipe_in(input.clone()) + .succeeds() + .stdout_is_bytes(input); +} + +#[test] +fn test_zero_width_bytes_in_character_mode() { + let len = io_buf_size_times_two(); + let input = vec![0u8; len]; + let expected = fold_characters_reference_bytes(&input, 80); + + new_ucmd!() + .args(&["--characters"]) + .pipe_in(input) + .succeeds() + .stdout_is_bytes(expected); +} + +#[test] +fn test_zero_width_spaces_in_column_mode() { + let len = io_buf_size_times_two(); + let input = "\u{200B}".repeat(len); + + new_ucmd!() + .pipe_in(input.clone()) + .succeeds() + .stdout_is(&input); +} + +#[test] +fn test_zero_width_spaces_in_character_mode() { + let len = io_buf_size_times_two(); + let input = "\u{200B}".repeat(len); + let expected = fold_characters_reference(&input, 80); + + new_ucmd!() + .args(&["--characters"]) + .pipe_in(input) + .succeeds() + .stdout_is(&expected); +} + +#[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "netbsd"))] +#[test] +fn test_fold_reports_no_space_left_on_dev_full() { + use std::fs::OpenOptions; + use std::process::Stdio; + + for &byte in &[b'\n', b'\0', 0xC3u8] { + let dev_full = OpenOptions::new() + .write(true) + .open("/dev/full") + .expect("/dev/full must exist on supported targets"); + + new_ucmd!() + .pipe_in(vec![byte; 1024]) + .set_stdout(Stdio::from(dev_full)) + .fails() + .stderr_contains("No space left"); + } +} + +fn buf_reader_capacity() -> usize { + std::io::BufReader::new(&b""[..]).capacity() +} + +fn io_buf_size_times_two() -> usize { + buf_reader_capacity() + .checked_mul(2) + .expect("BufReader capacity overflow") +} + +fn fold_characters_reference(input: &str, width: usize) -> String { + let mut output = String::with_capacity(input.len()); + let mut col_count = 0usize; + + for ch in input.chars() { + if ch == '\n' { + output.push('\n'); + col_count = 0; + continue; + } + + if col_count >= width { + output.push('\n'); + col_count = 0; + } + + output.push(ch); + col_count += 1; + } + + output +} + +fn fold_characters_reference_bytes(input: &[u8], width: usize) -> Vec { + let mut output = Vec::with_capacity(input.len() + input.len() / width + 1); + + for chunk in input.chunks(width) { + output.extend_from_slice(chunk); + if chunk.len() == width { + output.push(b'\n'); + } + } + + output +} + +fn tail_inclusive(text: &str, lines: usize) -> String { + if lines == 0 { + return String::new(); + } + + let segments: Vec<&str> = text.split_inclusive('\n').collect(); + if segments.is_empty() { + return text.to_owned(); + } + + let start = segments.len().saturating_sub(lines); + segments[start..].concat() +} + #[test] fn test_should_preserve_empty_line_without_final_newline() { new_ucmd!() From 556804c2373b566a4e0963dc17f5e5104821852d Mon Sep 17 00:00:00 2001 From: mattsu Date: Sat, 15 Nov 2025 08:57:58 +0900 Subject: [PATCH 07/15] refactor: clean up formatting in fold utility and tests - Remove trailing empty lines in fold.rs - Compact multiline variable assignments in test_fold.rs for readability --- src/uu/fold/src/fold.rs | 2 -- tests/by-util/test_fold.rs | 10 ++-------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/uu/fold/src/fold.rs b/src/uu/fold/src/fold.rs index 2ebfa65c248..713527a6f54 100644 --- a/src/uu/fold/src/fold.rs +++ b/src/uu/fold/src/fold.rs @@ -453,7 +453,6 @@ fn push_ascii_segment(segment: &[u8], ctx: &mut FoldContext<'_, W>) -> } remaining = &remaining[take..]; - } Ok(()) @@ -530,7 +529,6 @@ fn process_utf8_line(line: &str, ctx: &mut FoldContext<'_, W>) -> URes push_bytes(ctx, &line_bytes[byte_idx..next_idx])?; *ctx.col_count = ctx.col_count.saturating_add(added); - } Ok(()) diff --git a/tests/by-util/test_fold.rs b/tests/by-util/test_fold.rs index f15199fb612..f8d6e01cca2 100644 --- a/tests/by-util/test_fold.rs +++ b/tests/by-util/test_fold.rs @@ -111,15 +111,9 @@ fn test_unicode_on_reader_buffer_boundary_in_character_mode() { input.push_str(&"a".repeat(100)); input.push('\n'); - let expected_tail = tail_inclusive( - &fold_characters_reference(&input, 80), - 4, - ); + let expected_tail = tail_inclusive(&fold_characters_reference(&input, 80), 4); - let result = new_ucmd!() - .arg("--characters") - .pipe_in(input) - .succeeds(); + let result = new_ucmd!().arg("--characters").pipe_in(input).succeeds(); let actual_tail = tail_inclusive(result.stdout_str(), 4); From a1bbd4036a9126f7917550a5cd65af7ef333f870 Mon Sep 17 00:00:00 2001 From: mattsu Date: Sat, 15 Nov 2025 09:29:43 +0900 Subject: [PATCH 08/15] feat(fold): add unicode-width dependency and tests for zero-width characters Add unicode-width crate to handle zero-width Unicode characters in fold utility. Introduced new test 'test_zero_width_data_line_counts' to verify correct wrapping in --characters mode for zero-width bytes and spaces, ensuring fold behaves consistently with character counts rather than visual width. --- Cargo.lock | 1 + Cargo.toml | 1 + tests/by-util/test_fold.rs | 51 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index f2f8412cc31..7a709e92bc6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -608,6 +608,7 @@ dependencies = [ "tempfile", "textwrap", "time", + "unicode-width 0.2.2", "unindent", "uu_arch", "uu_base32", diff --git a/Cargo.toml b/Cargo.toml index 499eb87412c..3ffacf3ce9a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -543,6 +543,7 @@ regex.workspace = true sha1 = { workspace = true, features = ["std"] } tempfile.workspace = true time = { workspace = true, features = ["local-offset"] } +unicode-width.workspace = true unindent = "0.2.3" uutests.workspace = true uucore = { workspace = true, features = [ diff --git a/tests/by-util/test_fold.rs b/tests/by-util/test_fold.rs index f8d6e01cca2..da1265d1503 100644 --- a/tests/by-util/test_fold.rs +++ b/tests/by-util/test_fold.rs @@ -2,6 +2,7 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +use unicode_width::UnicodeWidthChar; use uutests::new_ucmd; #[test] @@ -188,6 +189,52 @@ fn test_zero_width_spaces_in_character_mode() { .stdout_is(&expected); } +#[test] +fn test_zero_width_data_line_counts() { + let len = io_buf_size_times_two(); + + let zero_bytes = vec![0u8; len]; + let column_bytes = new_ucmd!().pipe_in(zero_bytes.clone()).succeeds(); + assert_eq!( + newline_count(column_bytes.stdout()), + 0, + "fold should not wrap zero-width bytes in column mode", + ); + + let characters_bytes = new_ucmd!() + .args(&["--characters"]) + .pipe_in(zero_bytes) + .succeeds(); + assert_eq!( + newline_count(characters_bytes.stdout()), + len / 80, + "fold --characters should wrap zero-width bytes every 80 bytes", + ); + + if UnicodeWidthChar::width('\u{200B}') != Some(0) { + eprintln!("skip zero width space checks because width != 0"); + return; + } + + let zero_width_spaces = "\u{200B}".repeat(len); + let column_spaces = new_ucmd!().pipe_in(zero_width_spaces.clone()).succeeds(); + assert_eq!( + newline_count(column_spaces.stdout()), + 0, + "fold should keep zero-width spaces on a single line in column mode", + ); + + let characters_spaces = new_ucmd!() + .args(&["--characters"]) + .pipe_in(zero_width_spaces) + .succeeds(); + assert_eq!( + newline_count(characters_spaces.stdout()), + len / 80, + "fold --characters should wrap zero-width spaces every 80 characters", + ); +} + #[cfg(any(target_os = "linux", target_os = "freebsd", target_os = "netbsd"))] #[test] fn test_fold_reports_no_space_left_on_dev_full() { @@ -254,6 +301,10 @@ fn fold_characters_reference_bytes(input: &[u8], width: usize) -> Vec { output } +fn newline_count(bytes: &[u8]) -> usize { + bytes.iter().filter(|&&b| b == b'\n').count() +} + fn tail_inclusive(text: &str, lines: usize) -> String { if lines == 0 { return String::new(); From d176e651ab1ff7d420e4676346256ae85e2148ac Mon Sep 17 00:00:00 2001 From: mattsu Date: Sat, 15 Nov 2025 09:41:06 +0900 Subject: [PATCH 09/15] perf: use bytecount for efficient newline counting in fold tests - Add bytecount dependency to Cargo.toml and Cargo.lock - Refactor newline_count function in test_fold.rs to use bytecount::count instead of manual iteration for better performance --- Cargo.lock | 1 + Cargo.toml | 1 + tests/by-util/test_fold.rs | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 7a709e92bc6..5f2dc510921 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -582,6 +582,7 @@ name = "coreutils" version = "0.4.0" dependencies = [ "bincode", + "bytecount", "chrono", "clap", "clap_complete", diff --git a/Cargo.toml b/Cargo.toml index 3ffacf3ce9a..11d53043b64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -536,6 +536,7 @@ ctor.workspace = true filetime.workspace = true glob.workspace = true libc.workspace = true +bytecount.workspace = true num-prime.workspace = true pretty_assertions = "1.4.0" rand.workspace = true diff --git a/tests/by-util/test_fold.rs b/tests/by-util/test_fold.rs index da1265d1503..33374a0ebf2 100644 --- a/tests/by-util/test_fold.rs +++ b/tests/by-util/test_fold.rs @@ -2,6 +2,7 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. +use bytecount::count; use unicode_width::UnicodeWidthChar; use uutests::new_ucmd; @@ -302,7 +303,7 @@ fn fold_characters_reference_bytes(input: &[u8], width: usize) -> Vec { } fn newline_count(bytes: &[u8]) -> usize { - bytes.iter().filter(|&&b| b == b'\n').count() + count(bytes, b'\n') } fn tail_inclusive(text: &str, lines: usize) -> String { From dbad8db67b6218a064834345cee0f39c208197e9 Mon Sep 17 00:00:00 2001 From: mattsu Date: Sat, 15 Nov 2025 11:15:56 +0900 Subject: [PATCH 10/15] refactor: Handle zero-width bytes across buffer boundaries in fold Modify the fold implementation to process input in buffered chunks rather than line-by-line reading, ensuring correct handling of multi-byte characters split across buffer boundaries. Add process_pending_chunk function and new streaming logic to fold_file for better performance on large files. Update tests accordingly. --- src/uu/fold/src/fold.rs | 86 ++++++++++++++++++++++++++++---------- tests/by-util/test_fold.rs | 38 +++++++++++++++++ 2 files changed, 103 insertions(+), 21 deletions(-) diff --git a/src/uu/fold/src/fold.rs b/src/uu/fold/src/fold.rs index 713527a6f54..75fbba63c91 100644 --- a/src/uu/fold/src/fold.rs +++ b/src/uu/fold/src/fold.rs @@ -357,7 +357,7 @@ fn process_ascii_line(line: &[u8], ctx: &mut FoldContext<'_, W>) -> UR NL => { *ctx.last_space = None; emit_output(ctx)?; - break; + idx += 1; } CR => { push_byte(ctx, CR)?; @@ -472,7 +472,7 @@ fn process_utf8_line(line: &str, ctx: &mut FoldContext<'_, W>) -> URes if ch == '\n' { *ctx.last_space = None; emit_output(ctx)?; - break; + continue; } if *ctx.col_count >= ctx.width { @@ -539,7 +539,7 @@ fn process_non_utf8_line(line: &[u8], ctx: &mut FoldContext<'_, W>) -> if byte == NL { *ctx.last_space = None; emit_output(ctx)?; - break; + continue; } if *ctx.col_count >= ctx.width { @@ -576,6 +576,43 @@ fn process_non_utf8_line(line: &[u8], ctx: &mut FoldContext<'_, W>) -> Ok(()) } +fn process_pending_chunk( + pending: &mut Vec, + ctx: &mut FoldContext<'_, W>, +) -> UResult<()> { + loop { + if pending.is_empty() { + break; + } + + match std::str::from_utf8(pending) { + Ok(valid) => { + process_utf8_line(valid, ctx)?; + pending.clear(); + break; + } + Err(err) => { + if let Some(_) = err.error_len() { + process_non_utf8_line(pending, ctx)?; + pending.clear(); + break; + } else { + let valid_up_to = err.valid_up_to(); + if valid_up_to > 0 { + let valid = + std::str::from_utf8(&pending[..valid_up_to]).expect("valid prefix"); + process_utf8_line(valid, ctx)?; + pending.drain(..valid_up_to); + } + break; + } + } + } + } + + Ok(()) +} + /// Fold `file` to fit `width` (number of columns). /// /// By default `fold` treats tab, backspace, and carriage return specially: @@ -592,20 +629,12 @@ fn fold_file( mode: WidthMode, writer: &mut W, ) -> UResult<()> { - let mut line = Vec::new(); let mut output = Vec::new(); let mut col_count = 0; let mut last_space = None; + let mut pending = Vec::new(); - loop { - if file - .read_until(NL, &mut line) - .map_err_context(|| translate!("fold-error-readline"))? - == 0 - { - break; - } - + { let mut ctx = FoldContext { spaces, width, @@ -616,17 +645,32 @@ fn fold_file( last_space: &mut last_space, }; - match std::str::from_utf8(&line) { - Ok(s) => process_utf8_line(s, &mut ctx)?, - Err(_) => process_non_utf8_line(&line, &mut ctx)?, + loop { + let buffer = file + .fill_buf() + .map_err_context(|| translate!("fold-error-readline"))?; + if buffer.is_empty() { + break; + } + pending.extend_from_slice(buffer); + let consumed = buffer.len(); + file.consume(consumed); + + process_pending_chunk(&mut pending, &mut ctx)?; } - line.clear(); - } + if !pending.is_empty() { + match std::str::from_utf8(&pending) { + Ok(s) => process_utf8_line(s, &mut ctx)?, + Err(_) => process_non_utf8_line(&pending, &mut ctx)?, + } + pending.clear(); + } - if !output.is_empty() { - writer.write_all(&output)?; - output.clear(); + if !ctx.output.is_empty() { + ctx.writer.write_all(ctx.output)?; + ctx.output.clear(); + } } Ok(()) diff --git a/tests/by-util/test_fold.rs b/tests/by-util/test_fold.rs index 33374a0ebf2..b022a313e95 100644 --- a/tests/by-util/test_fold.rs +++ b/tests/by-util/test_fold.rs @@ -5,6 +5,8 @@ use bytecount::count; use unicode_width::UnicodeWidthChar; use uutests::new_ucmd; +use uutests::util::TestScenario; +use uutests::util_name; #[test] fn test_invalid_arg() { @@ -190,6 +192,42 @@ fn test_zero_width_spaces_in_character_mode() { .stdout_is(&expected); } +#[test] +fn test_zero_width_bytes_from_file() { + let len = io_buf_size_times_two(); + let input = vec![0u8; len]; + let expected = fold_characters_reference_bytes(&input, 80); + + let ts = TestScenario::new(util_name!()); + let path = "zeros.bin"; + ts.fixtures.write_bytes(path, &input); + + ts.ucmd().arg(path).succeeds().stdout_is_bytes(&input); + + ts.ucmd() + .args(&["--characters", path]) + .succeeds() + .stdout_is_bytes(expected); +} + +#[test] +fn test_zero_width_spaces_from_file() { + let len = io_buf_size_times_two(); + let input = "\u{200B}".repeat(len); + let expected = fold_characters_reference(&input, 80); + + let ts = TestScenario::new(util_name!()); + let path = "zero-width.txt"; + ts.fixtures.write(path, &input); + + ts.ucmd().arg(path).succeeds().stdout_is(&input); + + ts.ucmd() + .args(&["--characters", path]) + .succeeds() + .stdout_is(&expected); +} + #[test] fn test_zero_width_data_line_counts() { let len = io_buf_size_times_two(); From cd6f536bcdab4a56256f7bf4c2ffc6a2cb8819c8 Mon Sep 17 00:00:00 2001 From: mattsu Date: Sat, 15 Nov 2025 11:20:08 +0900 Subject: [PATCH 11/15] refactor(fold): streamline process_pending_chunk loop and error handling Replace loop with early empty check by a while loop conditional on !pending.is_empty() for clarity. Restructure invalid UTF-8 error handling to first check if valid_up_to == 0, then process the valid prefix, improving code readability and flow without changing behavior. --- src/uu/fold/src/fold.rs | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/uu/fold/src/fold.rs b/src/uu/fold/src/fold.rs index 75fbba63c91..2d131e8c974 100644 --- a/src/uu/fold/src/fold.rs +++ b/src/uu/fold/src/fold.rs @@ -580,11 +580,7 @@ fn process_pending_chunk( pending: &mut Vec, ctx: &mut FoldContext<'_, W>, ) -> UResult<()> { - loop { - if pending.is_empty() { - break; - } - + while !pending.is_empty() { match std::str::from_utf8(pending) { Ok(valid) => { process_utf8_line(valid, ctx)?; @@ -592,20 +588,21 @@ fn process_pending_chunk( break; } Err(err) => { - if let Some(_) = err.error_len() { + if err.error_len().is_some() { process_non_utf8_line(pending, ctx)?; pending.clear(); break; - } else { - let valid_up_to = err.valid_up_to(); - if valid_up_to > 0 { - let valid = - std::str::from_utf8(&pending[..valid_up_to]).expect("valid prefix"); - process_utf8_line(valid, ctx)?; - pending.drain(..valid_up_to); - } + } + + let valid_up_to = err.valid_up_to(); + if valid_up_to == 0 { break; } + + let valid = + std::str::from_utf8(&pending[..valid_up_to]).expect("valid prefix"); + process_utf8_line(valid, ctx)?; + pending.drain(..valid_up_to); } } } From 75c542f7d79278cb577291862fccb4346a466711 Mon Sep 17 00:00:00 2001 From: mattsu Date: Sat, 15 Nov 2025 11:21:05 +0900 Subject: [PATCH 12/15] refactor(fold): condense variable assignment to single line Consolidate the assignment of the `valid` variable from multiple lines to a single line for improved code readability and adherence to style guidelines favoring concise declarations. --- src/uu/fold/src/fold.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/uu/fold/src/fold.rs b/src/uu/fold/src/fold.rs index 2d131e8c974..a594ba03450 100644 --- a/src/uu/fold/src/fold.rs +++ b/src/uu/fold/src/fold.rs @@ -599,8 +599,7 @@ fn process_pending_chunk( break; } - let valid = - std::str::from_utf8(&pending[..valid_up_to]).expect("valid prefix"); + let valid = std::str::from_utf8(&pending[..valid_up_to]).expect("valid prefix"); process_utf8_line(valid, ctx)?; pending.drain(..valid_up_to); } From eb7d8c4f2ad86245f8bef28e343e97bc165fbbf3 Mon Sep 17 00:00:00 2001 From: mattsu Date: Thu, 20 Nov 2025 18:52:41 +0900 Subject: [PATCH 13/15] fix(fold): properly handle combining characters in character-counting mode Only coalesce zero-width combining characters into base characters when folding by display columns (WidthMode::Columns). In character-counting mode, treat every scalar value as advancing the counter to match chars().count() semantics, preventing incorrect line breaking for characters with zero-width marks. This ensures consistent behavior across modes as verified by existing tests. --- src/uu/fold/src/fold.rs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/uu/fold/src/fold.rs b/src/uu/fold/src/fold.rs index 35215da5006..b3bef85727a 100644 --- a/src/uu/fold/src/fold.rs +++ b/src/uu/fold/src/fold.rs @@ -467,12 +467,18 @@ fn process_utf8_line(line: &str, ctx: &mut FoldContext<'_, W>) -> URes let mut iter = line.char_indices().peekable(); while let Some((byte_idx, ch)) = iter.next() { - // Include combining characters with the base character - while let Some(&(_, next_ch)) = iter.peek() { - if unicode_width::UnicodeWidthChar::width(next_ch).unwrap_or(1) == 0 { - iter.next(); - } else { - break; + // Include combining characters with the base character when we are + // measuring by display columns. In character-counting mode every + // scalar value must advance the counter to match `chars().count()` + // semantics (see `fold_characters_reference` in the tests), so we do + // not coalesce zero-width scalars there. + if ctx.mode == WidthMode::Columns { + while let Some(&(_, next_ch)) = iter.peek() { + if unicode_width::UnicodeWidthChar::width(next_ch).unwrap_or(1) == 0 { + iter.next(); + } else { + break; + } } } From e54a74b3f6498e39f0d12e80ad785a89910c4769 Mon Sep 17 00:00:00 2001 From: mattsu Date: Mon, 22 Dec 2025 21:02:25 +0900 Subject: [PATCH 14/15] refactor(fold): add comments to explain streaming flush logic and prevent unbounded buffering Add explanatory comments to constants and functions in fold.rs, detailing the 8 KiB threshold for flushing in streaming mode to avoid unbounded buffer growth, and clarifying line folding behavior with the -s option. Improves code readability without altering functionality. --- src/uu/fold/src/fold.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/uu/fold/src/fold.rs b/src/uu/fold/src/fold.rs index 38c17570420..3acea901682 100644 --- a/src/uu/fold/src/fold.rs +++ b/src/uu/fold/src/fold.rs @@ -19,6 +19,7 @@ const TAB_WIDTH: usize = 8; const NL: u8 = b'\n'; const CR: u8 = b'\r'; const TAB: u8 = b'\t'; +// Implementation threshold (8 KiB) to prevent unbounded buffer growth during streaming. const STREAMING_FLUSH_THRESHOLD: usize = 8 * 1024; mod options { @@ -289,6 +290,8 @@ fn compute_col_count(buffer: &[u8], mode: WidthMode) -> usize { } fn emit_output(ctx: &mut FoldContext<'_, W>) -> UResult<()> { + // Emit a folded line and keep the remaining buffer (if any) for the next line. + // When `-s` is active, we prefer breaking at the last recorded whitespace. let consume = match *ctx.last_space { Some(index) => index + 1, None => ctx.output.len(), @@ -324,6 +327,9 @@ fn emit_output(ctx: &mut FoldContext<'_, W>) -> UResult<()> { } fn maybe_flush_unbroken_output(ctx: &mut FoldContext<'_, W>) -> UResult<()> { + // In streaming mode without `-s`, avoid unbounded buffering by periodically + // flushing long unbroken output segments. When `-s` is enabled we must keep + // the buffer to preserve the last whitespace boundary for folding. if ctx.spaces || ctx.output.len() < STREAMING_FLUSH_THRESHOLD { return Ok(()); } @@ -336,11 +342,13 @@ fn maybe_flush_unbroken_output(ctx: &mut FoldContext<'_, W>) -> UResul } fn push_byte(ctx: &mut FoldContext<'_, W>, byte: u8) -> UResult<()> { + // Append a single byte and flush if the buffer grows too large. ctx.output.push(byte); maybe_flush_unbroken_output(ctx) } fn push_bytes(ctx: &mut FoldContext<'_, W>, bytes: &[u8]) -> UResult<()> { + // Append a byte slice and flush if the buffer grows too large. if bytes.is_empty() { return Ok(()); } From 5de4b281837d33b6dd840c961b2cfd442d221b24 Mon Sep 17 00:00:00 2001 From: mattsu Date: Mon, 22 Dec 2025 21:33:42 +0900 Subject: [PATCH 15/15] refactor(fold): extract UTF-8 char processing and improve pending chunk handling - Extract UTF-8 character processing into a new `process_utf8_chars` function for better code organization - Add documentation to `process_pending_chunk` explaining its behavior with buffered bytes and invalid UTF-8 - Modify `process_pending_chunk` to properly handle the result of `process_non_utf8_line`, ensuring errors are propagated after clearing the buffer --- src/uu/fold/src/fold.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/uu/fold/src/fold.rs b/src/uu/fold/src/fold.rs index 3acea901682..00b3a45cc77 100644 --- a/src/uu/fold/src/fold.rs +++ b/src/uu/fold/src/fold.rs @@ -471,6 +471,10 @@ fn process_utf8_line(line: &str, ctx: &mut FoldContext<'_, W>) -> URes return process_ascii_line(line.as_bytes(), ctx); } + process_utf8_chars(line, ctx) +} + +fn process_utf8_chars(line: &str, ctx: &mut FoldContext<'_, W>) -> UResult<()> { let line_bytes = line.as_bytes(); let mut iter = line.char_indices().peekable(); @@ -599,6 +603,11 @@ fn process_non_utf8_line(line: &[u8], ctx: &mut FoldContext<'_, W>) -> Ok(()) } +/// Process buffered bytes, emitting output for valid UTF-8 prefixes and +/// deferring incomplete sequences until more input arrives. +/// +/// If the buffer contains invalid UTF-8, it is handled in non-UTF-8 mode and +/// the buffer is fully consumed. fn process_pending_chunk( pending: &mut Vec, ctx: &mut FoldContext<'_, W>, @@ -612,8 +621,9 @@ fn process_pending_chunk( } Err(err) => { if err.error_len().is_some() { - process_non_utf8_line(pending, ctx)?; + let res = process_non_utf8_line(pending, ctx); pending.clear(); + res?; break; }