Skip to content

sha2: refactor backends #802

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
newpavlov wants to merge 1 commit into
base: master
Choose a base branch
from
+711 −682
Open

sha2: refactor backends #802

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 9 additions & 9 deletions .github/workflows/sha2.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,7 @@ jobs:
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft"
- run: cargo test --all-features
env:
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft-compact"
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft" --cfg sha2_backend_soft="compact"

# macOS tests
macos:
Expand All @@ -112,7 +112,7 @@ jobs:
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft"
- run: cargo test --all-features
env:
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft-compact"
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft" --cfg sha2_backend_soft="compact"

# Windows tests
windows:
Expand All @@ -139,7 +139,7 @@ jobs:
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft"
- run: cargo test --all-features
env:
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft-compact"
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft" --cfg sha2_backend_soft="compact"

# Cross-compiled tests
cross:
Expand Down Expand Up @@ -186,16 +186,16 @@ jobs:
toolchain: nightly
- run: cross test --package sha2 --all-features --target riscv64gc-unknown-linux-gnu
env:
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft" -C target-feature=+zknh,+zbkb
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft"
- run: cross test --package sha2 --all-features --target riscv64gc-unknown-linux-gnu
env:
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft-compact" -C target-feature=+zknh,+zbkb
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft" --cfg sha2_backend_soft="compact"
- run: cross test --package sha2 --all-features --target riscv64gc-unknown-linux-gnu
env:
RUSTFLAGS: -Dwarnings --cfg sha2_backend="riscv-zknh" -C target-feature=+zknh,+zbkb
- run: cross test --package sha2 --all-features --target riscv64gc-unknown-linux-gnu
env:
RUSTFLAGS: -Dwarnings --cfg sha2_backend="riscv-zknh-compact" -C target-feature=+zknh,+zbkb
RUSTFLAGS: -Dwarnings --cfg sha2_backend="riscv-zknh" --cfg sha2_backend_riscv_zknh="compact" -C target-feature=+zknh,+zbkb

riscv32-zknh:
runs-on: ubuntu-latest
Expand All @@ -208,16 +208,16 @@ jobs:
components: rust-src
- run: cargo build --all-features --target riscv32gc-unknown-linux-gnu -Z build-std
env:
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft" -C target-feature=+zknh,+zbkb
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft"
- run: cargo build --all-features --target riscv32gc-unknown-linux-gnu -Z build-std
env:
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft-compact" -C target-feature=+zknh,+zbkb
RUSTFLAGS: -Dwarnings --cfg sha2_backend="soft" --cfg sha2_backend_soft="compact"
- run: cargo build --all-features --target riscv32gc-unknown-linux-gnu -Z build-std
env:
RUSTFLAGS: -Dwarnings --cfg sha2_backend="riscv-zknh" -C target-feature=+zknh,+zbkb
- run: cargo build --all-features --target riscv32gc-unknown-linux-gnu -Z build-std
env:
RUSTFLAGS: -Dwarnings --cfg sha2_backend="riscv-zknh-compact" -C target-feature=+zknh,+zbkb
RUSTFLAGS: -Dwarnings --cfg sha2_backend="riscv-zknh" --cfg sha2_backend_riscv_zknh="compact" -C target-feature=+zknh,+zbkb

# wasmtime tests
wasm:
Expand Down
4 changes: 3 additions & 1 deletion sha2/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ oid = ["digest/oid"]
[lints.rust.unexpected_cfgs]
level = "warn"
check-cfg = [
'cfg(sha2_backend, values("soft", "soft-compact", "riscv-zknh", "riscv-zknh-compact"))',
'cfg(sha2_backend, values("aarch64-sha2", "aarch64-sha3", "soft", "riscv-zknh", "riscv-zknh-compact", "x86-avx2", "x86-shani"))',
'cfg(sha2_backend_soft, values("compact"))',
'cfg(sha2_backend_riscv_zknh, values("compact"))',
]

[package.metadata.docs.rs]
Expand Down
94 changes: 67 additions & 27 deletions sha2/src/sha256.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,31 +2,53 @@ cfg_if::cfg_if! {
if #[cfg(sha2_backend = "soft")] {
mod soft;
use soft::compress;
} else if #[cfg(sha2_backend = "soft-compact")] {
mod soft_compact;
use soft_compact::compress;
} else if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] {
mod soft;
} else if #[cfg(all(
any(target_arch = "x86", target_arch = "x86_64"),
sha2_backend = "x86-shani",
))] {
#[cfg(not(all(
target_feature = "sha",
target_feature = "sse2",
target_feature = "ssse3",
target_feature = "sse4.1",
)))]
compile_error!("x86-shani backend requires sha, sse2, ssse3, sse4.1 target features");

mod x86_shani;
use x86_shani::compress;

fn compress(state: &mut [u32; 8], blocks: &[[u8; 64]]) {
// SAFETY: we checked above that the required target features are enabled
unsafe { x86_shani::compress(state, blocks) }
}
} else if #[cfg(all(
any(target_arch = "riscv32", target_arch = "riscv64"),
sha2_backend = "riscv-zknh"
any(sha2_backend = "riscv-zknh", sha2_backend = "riscv-zknh-compact"),
))] {
#[cfg(not(all(
target_feature = "zknh",
any(target_feature = "zbb", target_feature = "zbkb")
)))]
compile_error!("riscv-zknh backend requires zknh and zbkb (or zbb) target features");

mod riscv_zknh;
mod riscv_zknh_utils;
use riscv_zknh::compress;

fn compress(state: &mut [u32; 8], blocks: &[[u8; 64]]) {
// SAFETY: we checked above that the required target features are enabled
unsafe { riscv_zknh::compress(state, blocks) }
}
} else if #[cfg(all(
any(target_arch = "riscv32", target_arch = "riscv64"),
sha2_backend = "riscv-zknh-compact"
target_arch = "aarch64",
sha2_backend = "aarch64-sha2",
))] {
mod riscv_zknh_compact;
mod riscv_zknh_utils;
use riscv_zknh_compact::compress;
} else if #[cfg(target_arch = "aarch64")] {
mod soft;
#[cfg(not(target_feature = "sha2"))]
compile_error!("aarch64-sha2 backend requires sha2 target feature");

mod aarch64_sha2;
use aarch64_sha2::compress;

fn compress(state: &mut [u64; 8], blocks: &[[u8; 128]]) {
// SAFETY: we checked above that the required target features are enabled
unsafe { aarch64_sha2::compress(state, blocks) }
}
} else if #[cfg(target_arch = "loongarch64")] {
mod loongarch64_asm;
use loongarch64_asm::compress;
Expand All @@ -35,17 +57,35 @@ cfg_if::cfg_if! {
use wasm32_simd128::compress;
} else {
mod soft;
use soft::compress;
}
}

#[inline(always)]
#[allow(dead_code)]
fn to_u32s(block: &[u8; 64]) -> [u32; 16] {
core::array::from_fn(|i| {
let chunk = block[4 * i..][..4].try_into().unwrap();
u32::from_be_bytes(chunk)
})
cfg_if::cfg_if! {
if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] {
mod x86_shani;
cpufeatures::new!(shani_cpuid, "sha", "sse2", "ssse3", "sse4.1");
} else if #[cfg(target_arch = "aarch64")] {
mod aarch64_sha2;
cpufeatures::new!(sha2_hwcap, "sha2");
}
}

fn compress(state: &mut [u32; 8], blocks: &[[u8; 64]]) {
cfg_if::cfg_if! {
if #[cfg(any(target_arch = "x86", target_arch = "x86_64"))] {
if shani_cpuid::get() {
// SAFETY: we checked that required target features are available
return unsafe { x86_shani::compress(state, blocks) };
}
} else if #[cfg(target_arch = "aarch64")] {
if sha2_hwcap::get() {
// SAFETY: we checked that `sha2` target feature is available
return unsafe { aarch64_sha2::compress(state, blocks) };
}
}
}

soft::compress(state, blocks);
}
}
}

/// Raw SHA-256 compression function.
Expand Down
21 changes: 4 additions & 17 deletions sha2/src/sha256/aarch64_sha2.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,26 +1,13 @@
//! SHA-256 `aarch64` backend.
//!
//! Implementation adapted from mbedtls.
#![allow(unsafe_op_in_unsafe_fn)]

// Implementation adapted from mbedtls.

use core::arch::aarch64::*;

use crate::consts::K32;

cpufeatures::new!(sha2_hwcap, "sha2");

pub(super) fn compress(state: &mut [u32; 8], blocks: &[[u8; 64]]) {
// TODO: Replace with https://github.com/rust-lang/rfcs/pull/2725
// after stabilization
if sha2_hwcap::get() {
unsafe { sha256_compress(state, blocks) }
} else {
super::soft::compress(state, blocks);
}
}
use core::arch::aarch64::*;

#[target_feature(enable = "sha2")]
unsafe fn sha256_compress(state: &mut [u32; 8], blocks: &[[u8; 64]]) {
pub(super) unsafe fn compress(state: &mut [u32; 8], blocks: &[[u8; 64]]) {
// SAFETY: Requires the sha2 feature.

// Load state into vectors.
Expand Down
114 changes: 11 additions & 103 deletions sha2/src/sha256/riscv_zknh.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,108 +1,16 @@
use crate::consts::K32;
mod utils;

#[cfg(target_arch = "riscv32")]
use core::arch::riscv32::*;
use core::arch::riscv32::{sha256sig0, sha256sig1, sha256sum0, sha256sum1};
#[cfg(target_arch = "riscv64")]
use core::arch::riscv64::*;

#[cfg(not(all(
target_feature = "zknh",
any(target_feature = "zbb", target_feature = "zbkb")
)))]
compile_error!("riscv-zknh backend requires zknh and zbkb (or zbb) target features");

#[inline(always)]
fn ch(x: u32, y: u32, z: u32) -> u32 {
(x & y) ^ (!x & z)
}

#[inline(always)]
fn maj(x: u32, y: u32, z: u32) -> u32 {
(x & y) ^ (x & z) ^ (y & z)
}

fn round<const R: usize>(state: &mut [u32; 8], block: &[u32; 16], k: &[u32]) {
let n = K32.len() - R;
#[allow(clippy::identity_op)]
let a = (n + 0) % 8;
let b = (n + 1) % 8;
let c = (n + 2) % 8;
let d = (n + 3) % 8;
let e = (n + 4) % 8;
let f = (n + 5) % 8;
let g = (n + 6) % 8;
let h = (n + 7) % 8;

state[h] = state[h]
.wrapping_add(unsafe { sha256sum1(state[e]) })
.wrapping_add(ch(state[e], state[f], state[g]))
.wrapping_add(super::riscv_zknh_utils::opaque_load::<R>(k))
.wrapping_add(block[R]);
state[d] = state[d].wrapping_add(state[h]);
state[h] = state[h]
.wrapping_add(unsafe { sha256sum0(state[a]) })
.wrapping_add(maj(state[a], state[b], state[c]))
}

fn round_schedule<const R: usize>(state: &mut [u32; 8], block: &mut [u32; 16], k: &[u32]) {
round::<R>(state, block, k);

block[R] = block[R]
.wrapping_add(unsafe { sha256sig1(block[(R + 14) % 16]) })
.wrapping_add(block[(R + 9) % 16])
.wrapping_add(unsafe { sha256sig0(block[(R + 1) % 16]) });
}

#[inline(always)]
fn compress_block(state: &mut [u32; 8], mut block: [u32; 16]) {
let s = &mut state.clone();
let b = &mut block;

for i in 0..3 {
let k = &K32[16 * i..];
round_schedule::<0>(s, b, k);
round_schedule::<1>(s, b, k);
round_schedule::<2>(s, b, k);
round_schedule::<3>(s, b, k);
round_schedule::<4>(s, b, k);
round_schedule::<5>(s, b, k);
round_schedule::<6>(s, b, k);
round_schedule::<7>(s, b, k);
round_schedule::<8>(s, b, k);
round_schedule::<9>(s, b, k);
round_schedule::<10>(s, b, k);
round_schedule::<11>(s, b, k);
round_schedule::<12>(s, b, k);
round_schedule::<13>(s, b, k);
round_schedule::<14>(s, b, k);
round_schedule::<15>(s, b, k);
}

let k = &K32[48..];
round::<0>(s, b, k);
round::<1>(s, b, k);
round::<2>(s, b, k);
round::<3>(s, b, k);
round::<4>(s, b, k);
round::<5>(s, b, k);
round::<6>(s, b, k);
round::<7>(s, b, k);
round::<8>(s, b, k);
round::<9>(s, b, k);
round::<10>(s, b, k);
round::<11>(s, b, k);
round::<12>(s, b, k);
round::<13>(s, b, k);
round::<14>(s, b, k);
round::<15>(s, b, k);

for i in 0..8 {
state[i] = state[i].wrapping_add(s[i]);
}
}

pub(super) fn compress(state: &mut [u32; 8], blocks: &[[u8; 64]]) {
for block in blocks.iter().map(super::riscv_zknh_utils::load_block) {
compress_block(state, block);
use core::arch::riscv64::{sha256sig0, sha256sig1, sha256sum0, sha256sum1};

cfg_if::cfg_if! {
if #[cfg(sha2_backend_riscv_zknh = "compact")] {
mod compact;
pub(super) use compact::compress;
} else {
mod unroll;
pub(super) use unroll::compress;
}
}
Loading
Loading