diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 2b519a989f3..833a5799dce 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -53,7 +53,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -64,7 +64,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -184,9 +184,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.19.0" +version = "3.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" [[package]] name = "bytecount" @@ -196,9 +196,9 @@ checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" [[package]] name = "cc" -version = "1.2.48" +version = "1.2.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c481bdbf0ed3b892f6f806287d72acd515b352a4ec27a208489b8c1bc839633a" +checksum = "9f50d563227a1c37cc0a263f64eca3334388c01c5e4c4861a9def205c614383c" dependencies = [ "find-msvc-tools", "jobserver", @@ -325,9 +325,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.4.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" dependencies = [ "crc-catalog", ] @@ -340,9 +340,9 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc-fast" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2f7c8d397a6353ef0c1d6217ab91b3ddb5431daf57fd013f506b967dcf44458" +checksum = "2fd92aca2c6001b1bf5ba0ff84ee74ec8501b52bbef0cac80bf25a6c1d87a83d" dependencies = [ "crc", "digest", @@ -504,7 +504,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -519,6 +519,17 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" +[[package]] +name = "fixed_decimal" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35eabf480f94d69182677e37571d3be065822acfafd12f2f085db44fbbcc8e57" +dependencies = [ + "displaydoc", + "smallvec", + "writeable", +] + [[package]] name = "flate2" version = "1.1.5" @@ -693,6 +704,27 @@ dependencies = [ "zerovec", ] +[[package]] +name = "icu_decimal" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a38c52231bc348f9b982c1868a2af3195199623007ba2c7650f432038f5b3e8e" +dependencies = [ + "fixed_decimal", + "icu_decimal_data", + "icu_locale", + "icu_locale_core", + "icu_provider", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_decimal_data" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2905b4044eab2dd848fe84199f9195567b63ab3a93094711501363f63546fef7" + [[package]] name = "icu_locale" version = "2.1.1" @@ -753,9 +785,9 @@ checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" +checksum = "020bfc02fe870ec3a66d93e677ccca0562506e5872c650f893269e08615d74ec" dependencies = [ "icu_collections", "icu_locale_core", @@ -767,9 +799,9 @@ dependencies = [ [[package]] name = "icu_properties_data" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" +checksum = "616c294cf8d725c6afcd8f55abc17c56464ef6211f9ed59cccffe534129c77af" [[package]] name = "icu_provider" @@ -824,9 +856,9 @@ dependencies = [ [[package]] name = "jiff" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35" +checksum = "a87d9b8105c23642f50cbbae03d1f75d8422c5cb98ce7ee9271f7ff7505be6b8" dependencies = [ "jiff-static", "jiff-tzdb-platform", @@ -834,14 +866,14 @@ dependencies = [ "portable-atomic", "portable-atomic-util", "serde_core", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "jiff-static" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" +checksum = "b787bebb543f8969132630c51fd0afab173a86c6abae56ff3b9e5e3e3f9f6e58" dependencies = [ "proc-macro2", "quote", @@ -850,9 +882,9 @@ dependencies = [ [[package]] name = "jiff-tzdb" -version = "0.1.4" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1283705eb0a21404d2bfd6eef2a7593d240bc42a0bdb39db0ad6fa2ec026524" +checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2" [[package]] name = "jiff-tzdb-platform" @@ -1119,9 +1151,9 @@ checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" [[package]] name = "portable-atomic" -version = "1.11.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" +checksum = "f59e70c4aef1e55797c2e8fd94a4f2a973fc972cfde0e0b05f683667b0cd39dd" [[package]] name = "portable-atomic-util" @@ -1273,15 +1305,15 @@ checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" [[package]] name = "rustix" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1366,9 +1398,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "simd-adler32" -version = "0.3.7" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" [[package]] name = "similar" @@ -1439,15 +1471,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.23.0" +version = "3.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" dependencies = [ "fastrand", "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1748,7 +1780,9 @@ dependencies = [ "glob", "hex", "icu_collator", + "icu_decimal", "icu_locale", + "icu_provider", "itertools", "libc", "md-5", @@ -1902,7 +1936,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] diff --git a/src/uu/sort/Cargo.toml b/src/uu/sort/Cargo.toml index 184f6776be7..e0f31c3c18c 100644 --- a/src/uu/sort/Cargo.toml +++ b/src/uu/sort/Cargo.toml @@ -34,7 +34,12 @@ self_cell = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } unicode-width = { workspace = true } -uucore = { workspace = true, features = ["fs", "parser-size", "version-cmp"] } +uucore = { workspace = true, features = [ + "fs", + "parser-size", + "version-cmp", + "i18n-decimal", +] } fluent = { workspace = true } [target.'cfg(unix)'.dependencies] diff --git a/src/uu/sort/src/sort.rs b/src/uu/sort/src/sort.rs index 65ab9911b92..34d41be0686 100644 --- a/src/uu/sort/src/sort.rs +++ b/src/uu/sort/src/sort.rs @@ -45,6 +45,7 @@ use uucore::error::{FromIo, strip_errno}; use uucore::error::{UError, UResult, USimpleError, UUsageError}; use uucore::extendedbigdecimal::ExtendedBigDecimal; use uucore::format_usage; +use uucore::i18n::decimal::locale_decimal_separator; use uucore::line_ending::LineEnding; use uucore::parser::num_parser::{ExtendedParser, ExtendedParserError}; use uucore::parser::parse_size::{ParseSizeError, Parser}; @@ -113,6 +114,24 @@ mod options { const DECIMAL_PT: u8 = b'.'; +fn locale_decimal_pt() -> u8 { + match locale_decimal_separator().as_bytes().first().copied() { + Some(b'.') => b'.', + Some(b',') => b',', + _ => DECIMAL_PT, + } +} + +fn effective_decimal_pt(input: &[u8], locale_decimal: u8) -> u8 { + if locale_decimal == b',' { + let has_comma = input.contains(&b','); + if !has_comma && input.contains(&b'.') { + return b'.'; + } + } + locale_decimal +} + const NEGATIVE: &u8 = &b'-'; const POSITIVE: &u8 = &b'+'; @@ -637,8 +656,9 @@ impl<'a> Line<'a> { } SortMode::GeneralNumeric => { let initial_selection = &self.line[selection.clone()]; - - let leading = get_leading_gen(initial_selection); + let locale_decimal = locale_decimal_pt(); + let decimal_pt = effective_decimal_pt(initial_selection, locale_decimal); + let leading = get_leading_gen(initial_selection, decimal_pt); // Shorten selection to leading. selection.start += leading.start; @@ -965,7 +985,12 @@ impl FieldSelector { Selection::WithNumInfo(range_str, info) } else if self.settings.mode == SortMode::GeneralNumeric { // Parse this number as BigDecimal, as this is the requirement for general numeric sorting. - Selection::AsBigDecimal(general_bd_parse(&range_str[get_leading_gen(range_str)])) + let locale_decimal = locale_decimal_pt(); + let decimal_pt = effective_decimal_pt(range_str, locale_decimal); + Selection::AsBigDecimal(general_bd_parse( + &range_str[get_leading_gen(range_str, decimal_pt)], + decimal_pt, + )) } else { // This is not a numeric sort, so we don't need a NumCache. Selection::Str(range_str) @@ -2020,7 +2045,7 @@ fn ascii_case_insensitive_cmp(a: &[u8], b: &[u8]) -> Ordering { // scientific notation, so we strip those lines only after the end of the following numeric string. // For example, 5e10KFD would be 5e10 or 5x10^10 and +10000HFKJFK would become 10000. #[allow(clippy::cognitive_complexity)] -fn get_leading_gen(inp: &[u8]) -> Range { +fn get_leading_gen(inp: &[u8], decimal_pt: u8) -> Range { let trimmed = inp.trim_ascii_start(); let leading_whitespace_len = inp.len() - trimmed.len(); @@ -2058,7 +2083,7 @@ fn get_leading_gen(inp: &[u8]) -> Range { continue; } - if c == DECIMAL_PT && !had_decimal_pt && !had_e_notation { + if c == decimal_pt && !had_decimal_pt && !had_e_notation { had_decimal_pt = true; continue; } @@ -2101,9 +2126,16 @@ pub enum GeneralBigDecimalParseResult { /// Parse the beginning string into a [`GeneralBigDecimalParseResult`]. /// Using a [`GeneralBigDecimalParseResult`] instead of [`ExtendedBigDecimal`] is necessary to correctly order floats. #[inline(always)] -fn general_bd_parse(a: &[u8]) -> GeneralBigDecimalParseResult { +fn general_bd_parse(a: &[u8], decimal_pt: u8) -> GeneralBigDecimalParseResult { + let parsed_bytes = (decimal_pt != DECIMAL_PT).then(|| { + a.iter() + .map(|&b| if b == decimal_pt { DECIMAL_PT } else { b }) + .collect::>() + }); + let input = parsed_bytes.as_deref().unwrap_or(a); + // The string should be valid ASCII to be parsed. - let Ok(a) = std::str::from_utf8(a) else { + let Ok(a) = std::str::from_utf8(input) else { return GeneralBigDecimalParseResult::Invalid; }; diff --git a/tests/by-util/test_sort.rs b/tests/by-util/test_sort.rs index 99d388da0f1..18dec73db4a 100644 --- a/tests/by-util/test_sort.rs +++ b/tests/by-util/test_sort.rs @@ -1559,6 +1559,32 @@ fn test_g_float() { .stdout_is(output); } +#[test] +fn test_g_float_locale_decimal_separator() { + let Ok(locale_fr_utf8) = env::var("LOCALE_FR_UTF8") else { + return; + }; + if locale_fr_utf8 == "none" { + return; + } + + let ts = TestScenario::new("sort"); + + ts.ucmd() + .env("LC_ALL", &locale_fr_utf8) + .args(&["-g", "--stable"]) + .pipe_in("1,9\n1,10\n") + .succeeds() + .stdout_is("1,10\n1,9\n"); + + ts.ucmd() + .env("LC_ALL", &locale_fr_utf8) + .args(&["-g", "--stable"]) + .pipe_in("1.9\n1.10\n") + .succeeds() + .stdout_is("1.10\n1.9\n"); +} + #[test] // Test misc numbers ("'a" is not interpreted as literal, trailing text is ignored...) fn test_g_misc() {