Skip to content

wheel_architecture adjustments for 2023 #102

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
bartoldeman wants to merge 4 commits into
base: main
Choose a base branch
from
Draft

wheel_architecture adjustments for 2023 #102

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
df9bf4b
Under StdEnv/2023 make all wheels with C extensions gentoo2023
bartoldeman Oct 26, 2023
c5dfba1
Move all wheels with 2023 easybuild rpaths to arch dir.
bartoldeman Oct 26, 2023
3fb36c3
Added _has_avxes function to test for main AVXes instructions
ccoulombe Oct 26, 2023
a455f28
Added _has_bmis function to test for main BMI1 and BMI2 instructions
bartoldeman Oct 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions wheel_architecture.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@ cd $tmp
WORKS_ON_GENTOO2020=1
WORKS_ON_GENTOO2023=1
WORKS_ON_NIX=1
HAS_AVXES=0
HAS_BMIS=0

# On Nix we don't have Py 3.9 and newer so those wheels can be generic and don't need
# to be checked for Nix compatibility
Expand All @@ -31,6 +33,16 @@ function version_lte {
printf '%s\n%s' "$1" "$2" | sort -C -V && echo "yes" || echo "no"
}

function _has_avxes {
# This checks for any AVX instructions, that is : AVX, AVX2, or AVX512
objdump -D ${1:?missing file} | awk '/(vmovapd|vmulpd|vaddpd|vsubpd|vfmadd213pd|vfmadd231pd|vfmadd132pd|vmulsd|vaddsd|vmosd|vsubsd|vbroadcastss|vbroadcastsd|vblendpd|vshufpd|vroundpd|vroundsd|vxorpd|vfnmadd231pd|vfnmadd213pd|vfnmadd132pd|vandpd|vmaxpd|vmovmskpd|vcmppd|vpaddd|vbroadcastf128|vinsertf128|vextractf128|vfmsub231pd|vfmsub132pd|vfmsub213pd|vmaskmovps|vmaskmovpd|vpermilps|vpermilpd|vperm2f128|vzeroall|vzeroupper|vpbroadcastb|vpbroadcastw|vpbroadcastd|vpbroadcastq|vbroadcasti128|vinserti128|vextracti128|vpminud|vpmuludq|vgatherdpd|vgatherqpd|vgatherdps|vgatherqps|vpgatherdd|vpgatherdq|vpgatherqd|vpgatherqq|vpmaskmovd|vpmaskmovq|vpermps|vpermd|vpermpd|vpermq|vperm2i128|vpblendd|vpsllvd|vpsllvq|vpsrlvd|vpsrlvq|vpsravd|vblendmpd|vblendmps|vpblendmd|vpblendmq|vpblendmb|vpblendmw|vpcmpd|vpcmpud|vpcmpq|vpcmpuq|vpcmpb|vpcmpub|vpcmpw|vpcmpuw|vptestmd|vptestmq|vptestnmd|vptestnmq|vptestmb|vptestmw|vptestnmb|vptestnmw|vcompresspd|vcompressps|vpcompressd|vpcompressq|vexpandpd|vexpandps|vpexpandd|vpexpandq|vpermb|vpermw|vpermt2b|vpermt2w|vpermi2pd|vpermi2ps|vpermi2d|vpermi2q|vpermi2b|vpermi2w|vpermt2ps|vpermt2pd|vpermt2d|vpermt2q|vshuff32x4|vshuff64x2|vshuffi32x4|vshuffi64x2|vpmultishiftqb|vpternlogd|vpternlogq|vpmovqd|vpmovsqd|vpmovusqd|vpmovqw|vpmovsqw|vpmovusqw|vpmovqb|vpmovsqb|vpmovusqb|vpmovdw|vpmovsdw|vpmovusdw|vpmovdb|vpmovsdb|vpmovusdb|vpmovwb|vpmovswb|vpmovuswb|vcvtps2udq|vcvtpd2udq|vcvttps2udq|vcvttpd2udq|vcvtss2usi|vcvtsd2usi|vcvttss2usi|vcvttsd2usi|vcvtps2qq|vcvtpd2qq|vcvtps2uqq|vcvtpd2uqq|vcvttps2qq|vcvttpd2qq|vcvttps2uqq|vcvttpd2uqq|vcvtudq2ps|vcvtudq2pd|vcvtusi2ps|vcvtusi2pd|vcvtusi2sd|vcvtusi2ss|vcvtuqq2ps|vcvtuqq2pd|vcvtqq2pd|vcvtqq2ps|vgetexppd|vgetexpps|vgetexpsd|vgetexpss|vgetmantpd|vgetmantps|vgetmantsd|vgetmantss|vfixupimmpd|vfixupimmps|vfixupimmsd|vfixupimmss|vrcp14pd|vrcp14ps|vrcp14sd|vrcp14ss|vrndscaleps|vrndscalepd|vrndscaless|vrndscalesd|vrsqrt14pd|vrsqrt14ps|vrsqrt14sd|vrsqrt14ss|vscalefps|vscalefpd|vscalefss|vscalefsd|valignd|valignq|vdbpsadbw|vpabsq|vpmaxsq|vpmaxuq|vpminsq|vpminuq|vprold|vprolvd|vprolq|vprolvq|vprord|vprorvd|vprorq|vprorvq|vpscatterdd|vpscatterdq|vpscatterqd|vpscatterqq|vscatterdps|vscatterdpd|vscatterqps|vscatterqpd|vpconflictd|vpconflictq|vplzcntd|vplzcntq|vpbroadcastmb2q|vpbroadcastmw2d|vexp2pd|vexp2ps|vrcp28pd|vrcp28ps|vrcp28sd|vrcp28ss|vrsqrt28pd|vrsqrt28ps|vrsqrt28sd|vrsqrt28ss|vgatherpf0dps|vgatherpf0qps|vgatherpf0dpd|vgatherpf0qpd|vgatherpf1dps|vgatherpf1qps|vgatherpf1dpd|vgatherpf1qpd|vscatterpf0dps|vscatterpf0qps|vscatterpf0dpd|vscatterpf0qpd|vscatterpf1dps|vscatterpf1qps|vscatterpf1dpd|vscatterpf1qpd|vfpclassps|vfpclasspd|vfpclassss|vfpclasssd|vrangeps|vrangepd|vrangess|vrangesd|vreduceps|vreducepd|vreducess|vreducesd|vpmovm2d|vpmovm2q|vpmovm2b|vpmovm2w|vpmovd2m|vpmovq2m|vpmovb2m|vpmovw2m|vpmullq|vpmadd52luq|vpmadd52huq|v4fmaddps|v4fmaddss|v4fnmaddps|v4fnmaddss|vp4dpwssd|vp4dpwssds|vpdpbusd|vpdpbusds|vpdpwssd|vpdpwssds|vpcompressb|vpcompressw|vpexpandb|vpexpandw|vpshld|vpshldv|vpshrd|vpshrdv|vpopcntd|vpopcntq|vpopcntb|vpopcntw|vpshufbitqmb|gf2p8affineinvqb|gf2p8affineqb|gf2p8mulb|vpclmulqdq|vaesdec|vaesdeclast|vaesenc|vaesenclast|avx)/' | wc -l
}

function _has_bmis {
# This checks for any BMI instructions, that is : BMI1 and BMI2
objdump -D ${1:?missing file} | awk '/(andn|bextr|blsi|blsmsk|blsr|tzcnt|bzhi|mulx|pdep|pext|rorx|sarx|shrx|shlx)/' | wc -l
}

NIX_GLIBC="/cvmfs/soft.computecanada.ca/nix/var/nix/profiles/16.09/lib/libc.so.6"
GENTOO2020_GLIBC="/cvmfs/soft.computecanada.ca/gentoo/2020/lib64/libc.so.6"
GENTOO2023_GLIBC="/cvmfs/soft.computecanada.ca/gentoo/2023/x86-64-v3/lib64/libc.so.6"
Expand Down Expand Up @@ -65,6 +77,13 @@ for fname in $(find . -type f); do

# remove rpath from python itself from consideration
rpath=$(echo $rpath | sed -e "s;[^:]*/Core/python[^:]*;;g")

if [[ _has_avxes $fname -ge 1 ]]; then
HAS_AVXES=1
fi
if [[ _has_bmis $fname -ge 1 ]]; then
HAS_BMIS=1
fi

if [[ $WORKS_ON_NIX -eq 1 && $rpath =~ 'nix' || $rpath =~ 'easybuild/software/2017' ]]; then
WORKS_ON_GENTOO2020=0
Expand All @@ -74,7 +93,7 @@ for fname in $(find . -type f); do
WORKS_ON_NIX=0
WORKS_ON_GENTOO2023=0
echo "$fname" is gentoo2020, rpath=$rpath >&2
elif [[ $rpath =~ 'gentoo/2023' || $rpath =~ 'easybuild/software/2023' ]]; then
elif [[ $rpath =~ 'gentoo/2023' || $rpath =~ 'easybuild/software/2023' || "$EBVERSIONGENTOO" == "2023" ]]; then
WORKS_ON_NIX=0
WORKS_ON_GENTOO2020=0
echo "$fname" is gentoo2023, rpath=$rpath >&2
Expand Down Expand Up @@ -139,7 +158,7 @@ for fname in $(find . -type f); do
elif [[ $rpath =~ '/avx512/' ]]; then
ARCHITECTURE="avx512"
echo "$fname" is $COMPATIBILITY_LAYER $ARCHITECTURE, rpath=$rpath >&2
elif [[ $rpath =~ '/x86-64-v3/' && ! $rpath =~ '/x86-64-v3/Core/' ]]; then
elif [[ $rpath =~ '/x86-64-v3/' ]]; then
ARCHITECTURE="x86-64-v3"
echo "$fname" is $COMPATIBILITY_LAYER $ARCHITECTURE, rpath=$rpath >&2
elif [[ $rpath =~ '/x86-64-v4/' ]]; then
Expand Down