From d8d52a20d78187d2c21df580fcd2132ead40940b Mon Sep 17 00:00:00 2001 From: Clemens Lang Date: Sun, 11 Mar 2018 16:13:45 +0100 Subject: [PATCH 1/6] Revert "Only request variants explicitly when needed." This reverts commit f6e46815a5033358212ce383956fd567eb51e53c. --- mpbb-install-dependencies | 40 ++++++++++++++++----------------------- tools/dependencies.tcl | 20 +++++--------------- 2 files changed, 21 insertions(+), 39 deletions(-) diff --git a/mpbb-install-dependencies b/mpbb-install-dependencies index f83ec20..4ce2309 100644 --- a/mpbb-install-dependencies +++ b/mpbb-install-dependencies @@ -66,13 +66,10 @@ install-dependencies() { # Check whether any of the dependencies have previously failed failcachecounter=0 while read -r dependency; do - # Split portname +variant1+variant2 into portname and active - # variants, where the variants are optional. - set $dependency - depname="$1" - # remove surrounding quotes - depvariants="${2%\"}" - depvariants="${depvariants#\"}" + # Split portname +variant1+variant2 into portname and variants, where + # the variants are optional. + depname=${dependency%% *} + depvariants=${dependency:${#depname}+1} # $depvariants isn't quoted on purpose # shellcheck disable=SC2086 @@ -93,30 +90,25 @@ install-dependencies() { rm -f "${option_work_dir}/all_ports" while read -r dependency; do - # Split portname +variant1+variant2 into portname and active and - # requested variants, where the variants are optional. - set $dependency - depname=$1 - # remove surrounding quotes - depvariants="${2%\"}" - depvariants="${depvariants#\"}" - deprequestedvariants="${3%\"}" - deprequestedvariants="${deprequestedvariants#\"}" - - text="Installing dependency ($dependencies_counter of $dependencies_count) '${depname}' with variants '${depvariants}' (requesting '${deprequestedvariants}')" + # Split portname +variant1+variant2 into portname and variants, where + # the variants are optional. + depname=${dependency%% *} + depvariants=${dependency:${#depname}+1} + + text="Installing dependency ($dependencies_counter of $dependencies_count) '${depname}' with variants '${depvariants}'" echo "----> ${text}" echo -n "${text} ... " >> "$log_status_dependencies" # $option_prefix and $thisdir are set in mpbb # shellcheck disable=SC2154 - if [[ -f $("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/archive-path.tcl" "${depname}" "${deprequestedvariants}") ]]; then + if [[ -f $("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/archive-path.tcl" "${depname}" "${depvariants}") ]]; then echo "Already installed, nothing to do" echo "[OK]" >> "$log_status_dependencies" dependencies_counter=$((dependencies_counter + 1)) else - # $deprequestedvariants isn't quoted on purpose + # $depvariants isn't quoted on purpose # shellcheck disable=SC2154,SC2086 - if ! "${option_prefix}/bin/port" -dn install --unrequested "$depname" $deprequestedvariants; then - echo "Build of dependency '${depname}' with variants '${deprequestedvariants}' failed, aborting." >&2 + if ! "${option_prefix}/bin/port" -dn install --unrequested "$depname" $depvariants; then + echo "Build of dependency '${depname}' with variants '${depvariants}' failed, aborting." >&2 echo "[FAIL]" >> "$log_status_dependencies" echo "Building '$port' ... [ERROR] (failed to install dependency '${depname}') maintainers: $(get-maintainers "$port" "${depname}")." >> "$log_subports_progress" @@ -148,7 +140,7 @@ install-dependencies() { fi fi # add to the list for gather_archives - echo "$depname $deprequestedvariants" >> "${option_work_dir}/all_ports" + echo "$dependency" >> "${option_work_dir}/all_ports" done <<<"$dependencies" # activate everything now that we know it's all built and installed @@ -162,7 +154,7 @@ install-dependencies() { echo "Activating all dependencies..." # $option_prefix is set by mpbb, and dependencies isn't quoted on purpose # shellcheck disable=SC2154,SC2086 - if ! "${option_prefix}/bin/port" -dn install --unrequested $(cat "${option_work_dir}/all_ports"); then + if ! "${option_prefix}/bin/port" -dn install --unrequested ${dependencies}; then echo "Activating all dependencies failed, aborting." >&2 return 1 fi diff --git a/tools/dependencies.tcl b/tools/dependencies.tcl index fec0792..12a0aec 100755 --- a/tools/dependencies.tcl +++ b/tools/dependencies.tcl @@ -102,7 +102,7 @@ proc printdependency {ditem} { # Given the active_variants of the current dependency calculation and the # default variants, calculate the required string. set default_variants {} - if {[info exists depinfo(vinfo)]} { + if {[array size variants] > 0 && [info exists depinfo(vinfo)]} { foreach {vname vattrs} $depinfo(vinfo) { foreach {key val} $vattrs { if {$key eq "is_default" && $val eq "+"} { @@ -113,29 +113,19 @@ proc printdependency {ditem} { } } - set activevariantstring "" - set requestedvariantstring "" + set variantstring "" array set active_variants $depinfo(active_variants) set relevant_variants [lsort -unique [concat [array names active_variants] $default_variants]] foreach variant $relevant_variants { if {[info exists active_variants($variant)]} { - append activevariantstring "$active_variants($variant)$variant" - if {$variant ni $default_variants} { - append requestedvariantstring "$active_variants($variant)$variant" - } + append variantstring "$active_variants($variant)$variant" } else { # the only case where this situation can occur is a default variant that was explicitly disabled - append requestedvariantstring "-$variant" + append variantstring "-$variant" } } - if {$activevariantstring eq ""} { - set activevariantstring {""} - } - if {$requestedvariantstring eq ""} { - set requestedvariantstring {""} - } - puts [string trim "$depinfo(name) $activevariantstring $requestedvariantstring"] + puts [string trim "$depinfo(name) $variantstring"] } dlist_eval $dlist {} [list printdependency] From 548da36cd458d31b42a7eb1c98f9cec7ef039c19 Mon Sep 17 00:00:00 2001 From: Clemens Lang Date: Sun, 11 Mar 2018 16:37:12 +0100 Subject: [PATCH 2/6] Support specifying variants in mpbb install-port See: https://trac.macports.org/ticket/52742 --- mpbb-install-port | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/mpbb-install-port b/mpbb-install-port index 8fc0b72..b81decb 100644 --- a/mpbb-install-port +++ b/mpbb-install-port @@ -9,7 +9,7 @@ install-port-usage() { # "prog" is defined in mpbb-help. # shellcheck disable=SC2154 cat <] install-port +usage: $prog [] install-port [@(+|-)variant [...]] Build and install the given port. @@ -18,11 +18,12 @@ EOF } install-port() { - local port=${1-} - if [[ -z $port ]]; then + if [[ -z $1 ]]; then err "Must specify a port" return 1 fi + local port=${1%%@*} + local portvariants=${1:${#port}+1} # $option_log_dir is set in mpbb # shellcheck disable=SC2154 local log_port_contents="${option_log_dir}/port-contents.txt" @@ -37,7 +38,7 @@ install-port() { # $option_prefix and $thisdir are set in mpbb # shellcheck disable=SC2154 - if [[ -f $("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/archive-path.tcl" "$@") ]]; then + if [[ -f $("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/archive-path.tcl" "$port" "$portvariants") ]]; then echo "$* already installed, nothing to do" # log: summary for the portwatcher echo "Building '$port' ... [OK]" >> "$log_subports_progress" @@ -50,11 +51,10 @@ install-port() { time_start=$(date +%s) # $option_prefix is set in mpbb # shellcheck disable=SC2154 - if "${option_prefix}/bin/port" -dkn install "$@"; then + if "${option_prefix}/bin/port" -dkn install "$port" $portvariants; then # Remove failcache if it exists - failcache_success "$@" - if [ $? -ne 0 ]; then - err "failcache_success" "$@" "failed." + if ! failcache_success "$port" "$portvariants"; then + err "failcache_success $port $portvariants failed." return 1 fi else @@ -62,9 +62,8 @@ install-port() { # log: summary for the portwatcher echo "Building '$port' ... [ERROR] maintainers: $(get-maintainers "$port")." >> "$log_subports_progress" # update failcache - failcache_failure "$@" - if [ $? -ne 0 ]; then - err "failcache_failure" "$@" "failed." + if ! failcache_failure "$port" "$portvariants"; then + err "failcache_failure $port $portvariants failed." return 1 fi return 1 @@ -86,7 +85,7 @@ install-port() { local print_arg_workdir="ERROR" local print_arg_destdir="ERROR" # First, compute port_workdir_size and port_destdir_size - port_workdir=$("${option_prefix}/bin/port" work "$port") + port_workdir=$("${option_prefix}/bin/port" work "$port" $portvariants) if [ -n "$port_workdir" ]; then port_workdir_size=$(du -ks "$port_workdir" | sed 's/^ *//' | tr '\t' '\n' | head -n 1) if [ $? -eq 0 ] && [ -n "$port_workdir_size" ]; then @@ -111,7 +110,7 @@ install-port() { # log: main.log local port_mainlog - port_mainlog=$("${option_prefix}/bin/port" logfile "$port") + port_mainlog=$("${option_prefix}/bin/port" logfile "$port" $portvariants) if [ $? -eq 0 ] && [ -f "$port_mainlog" ]; then cp -f "$port_mainlog" "$log_port_main" fi From 778b151f8557e95d5b2353241ada9e33a9cfe752 Mon Sep 17 00:00:00 2001 From: Clemens Lang Date: Sun, 11 Mar 2018 16:37:45 +0100 Subject: [PATCH 3/6] Fix (or silence) install-port shellcheck warnings --- mpbb-install-port | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/mpbb-install-port b/mpbb-install-port index b81decb..a6e5fd5 100644 --- a/mpbb-install-port +++ b/mpbb-install-port @@ -34,7 +34,7 @@ install-port() { # prepare the log files and make sure to start with empty ones mkdir -p "${option_log_dir}" #> "$log_port_contents" - > "$log_port_stats" + :> "$log_port_stats" # $option_prefix and $thisdir are set in mpbb # shellcheck disable=SC2154 @@ -50,7 +50,7 @@ install-port() { local time_stop time_start=$(date +%s) # $option_prefix is set in mpbb - # shellcheck disable=SC2154 + # shellcheck disable=SC2154,SC2086 if "${option_prefix}/bin/port" -dkn install "$port" $portvariants; then # Remove failcache if it exists if ! failcache_success "$port" "$portvariants"; then @@ -85,9 +85,11 @@ install-port() { local print_arg_workdir="ERROR" local print_arg_destdir="ERROR" # First, compute port_workdir_size and port_destdir_size + # shellcheck disable=SC2086 port_workdir=$("${option_prefix}/bin/port" work "$port" $portvariants) if [ -n "$port_workdir" ]; then port_workdir_size=$(du -ks "$port_workdir" | sed 's/^ *//' | tr '\t' '\n' | head -n 1) + # shellcheck disable=SC2181 if [ $? -eq 0 ] && [ -n "$port_workdir_size" ]; then print_arg_workdir="${port_workdir_size}k" fi @@ -98,6 +100,7 @@ install-port() { print_arg_destdir="-" if [ -d "$port_destdir" ]; then port_destdir_size=$(du -ks "$port_destdir" | sed 's/^ *//' | tr '\t' '\n' | head -n 1) + # shellcheck disable=SC2181 if [ $? -eq 0 ] && [ -n "$port_destdir_size" ]; then print_arg_destdir="${port_destdir_size}k" fi @@ -110,7 +113,10 @@ install-port() { # log: main.log local port_mainlog + # portvariants is not quoted on purpose + # shellcheck disable=SC2086 port_mainlog=$("${option_prefix}/bin/port" logfile "$port" $portvariants) + # shellcheck disable=SC2181 if [ $? -eq 0 ] && [ -f "$port_mainlog" ]; then cp -f "$port_mainlog" "$log_port_main" fi From 50fad119c13a9165c1102d1b436712fb6a6b55a4 Mon Sep 17 00:00:00 2001 From: Clemens Lang Date: Sun, 11 Mar 2018 16:38:36 +0100 Subject: [PATCH 4/6] Support specifying variants in install-dependencies See: https://trac.macports.org/ticket/52742 --- mpbb-install-dependencies | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mpbb-install-dependencies b/mpbb-install-dependencies index 4ce2309..da76f46 100644 --- a/mpbb-install-dependencies +++ b/mpbb-install-dependencies @@ -9,7 +9,7 @@ install-dependencies-usage() { # "prog" is defined in mpbb-help. # shellcheck disable=SC2154 cat <] install-dependencies +usage: $prog [] install-dependencies [@(+|-)variant [...]] Build and install the dependencies of the given port. @@ -18,11 +18,12 @@ EOF } install-dependencies() { - local port=${1-} - if [[ -z $port ]]; then + if [[ -z $1 ]]; then err "Must specify a port" return 1 fi + local port=${1%%@*} + local portvariants=${1:${#port}+1} local dependencies local dependencies_count local dependencies_counter @@ -41,7 +42,7 @@ install-dependencies() { # calculate list of dependencies in-order # $option_prefix and $thisdir are set in mpbb # shellcheck disable=SC2154 - dependencies=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/dependencies.tcl" "$@") + dependencies=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/dependencies.tcl" "$port" "$portvariants") if [ $? -ne 0 ]; then echo "Calculating dependencies for '$port' failed, aborting." >&2 echo "Building '$port' ... [ERROR] (failed to calculate dependencies) maintainers: $(get-maintainers "$port")." >> "$log_subports_progress" From a156c41cbb7d110bfda14b3b44b1037e184a3d3a Mon Sep 17 00:00:00 2001 From: Clemens Lang Date: Sun, 11 Mar 2018 16:42:45 +0100 Subject: [PATCH 5/6] Fix (or silence) install-deps shellcheck warnings --- mpbb-install-dependencies | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/mpbb-install-dependencies b/mpbb-install-dependencies index da76f46..d962f3d 100644 --- a/mpbb-install-dependencies +++ b/mpbb-install-dependencies @@ -37,12 +37,13 @@ install-dependencies() { # prepare the log file and make sure to start with an empty one mkdir -p "${option_log_dir}" - > "$log_status_dependencies" + :> "$log_status_dependencies" # calculate list of dependencies in-order # $option_prefix and $thisdir are set in mpbb # shellcheck disable=SC2154 dependencies=$("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/dependencies.tcl" "$port" "$portvariants") + # shellcheck disable=SC2181 if [ $? -ne 0 ]; then echo "Calculating dependencies for '$port' failed, aborting." >&2 echo "Building '$port' ... [ERROR] (failed to calculate dependencies) maintainers: $(get-maintainers "$port")." >> "$log_subports_progress" @@ -62,7 +63,7 @@ install-dependencies() { echo >> "$log_status_dependencies" # save $@ since 'set' is used below - orig_args="$@" + orig_args="$*" # Check whether any of the dependencies have previously failed failcachecounter=0 @@ -88,6 +89,8 @@ install-dependencies() { return 1 fi + # option_work_dir is set in mpbb + # shellcheck disable=SC2154 rm -f "${option_work_dir}/all_ports" while read -r dependency; do @@ -116,8 +119,7 @@ install-dependencies() { # Update failcache # $depvariants isn't quoted on purpose # shellcheck disable=SC2086 - failcache_failure "$depname" $depvariants - if [ $? -ne 0 ]; then + if ! failcache_failure "$depname" $depvariants; then err "failcache_failure $depname $depvariants failed." return 1 fi @@ -128,8 +130,7 @@ install-dependencies() { # Remove failcache if it exists # $depvariants isn't quoted on purpose # shellcheck disable=SC2086 - failcache_success "$depname" $depvariants - if [ $? -ne 0 ]; then + if ! failcache_success "$depname" $depvariants; then err "failcache_success $depname $depvariants failed." return 1 fi @@ -147,8 +148,8 @@ install-dependencies() { # activate everything now that we know it's all built and installed # ... but only we're actually going to build the requested port - # $option_prefix and $thisdir are set in mpbb - # shellcheck disable=SC2154 + # $option_prefix and $thisdir are set in mpbb, orig_args is not quoted on purpose + # shellcheck disable=SC2154,SC2086 if [[ -f $("${option_prefix}/bin/port-tclsh" "${thisdir}/tools/archive-path.tcl" ${orig_args}) ]]; then echo "${orig_args} Already installed, not activating dependencies" else From 60af0776e80d1a4e96ff70bcc46847631d80d538 Mon Sep 17 00:00:00 2001 From: Clemens Lang Date: Sun, 11 Mar 2018 16:43:20 +0100 Subject: [PATCH 6/6] Support specifying variants in list-subports Allow passing variants into list-subports by separating port name and variants specification with an "@" sign. @ is not a valid character in port names and should allow us to specify variants per port in the buildbot's portlist. When not using this new syntax, the output will look exactly as it did before to avoid breaking the existing setup. Closes: https://trac.macports.org/ticket/52742 --- mpbb-list-subports | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/mpbb-list-subports b/mpbb-list-subports index 724d6aa..fe2ca26 100644 --- a/mpbb-list-subports +++ b/mpbb-list-subports @@ -9,7 +9,7 @@ list-subports-usage() { # "prog" is defined in mpbb-help. # shellcheck disable=SC2154 cat <] list-subports [] [ [...]] +usage: $prog [] list-subports [] [@(+|-)variant [...]] [[@(+|-)variant [...]] [...]] Print the name and subports of each given port to standard output. @@ -25,7 +25,8 @@ EOF print-subports() { local archive_site=$1 - local portname=$2 + local portname=${2%%@*} + local portvariants=${2:${#portname}+1} local port local portgroup local ports @@ -44,16 +45,15 @@ print-subports() { exclude=0 exclude_reasons=() - # FIXME: this doesn't take selected variants into account # $thisdir is set in mpbb # shellcheck disable=SC2154 - archive_path=$("${tclsh}" "${thisdir}/tools/archive-path.tcl" "${port}") + archive_path=$("${tclsh}" "${thisdir}/tools/archive-path.tcl" "${port}" "${portvariants}") if [[ -f "${archive_path}" ]]; then archive_basename=$(basename "${archive_path}") if curl -fIsL "${archive_site}/${port}/${archive_basename}" > /dev/null; then exclude=1 exclude_reasons+=("it has already been built and uploaded") - elif ! "${tclsh}" "${option_jobs_dir}/port_binary_distributable.tcl" "${port}"; then + elif ! "${tclsh}" "${option_jobs_dir}/port_binary_distributable.tcl" "${port}" "${portvariants}"; then exclude=1 exclude_reasons+=("it has already been built and is not distributable") fi @@ -66,7 +66,7 @@ print-subports() { else # $thisdir is set in mpbb # shellcheck disable=SC2154 - for portgroup in $("${tclsh}" "${thisdir}/tools/portgroups.tcl" "$port"); do + for portgroup in $("${tclsh}" "${thisdir}/tools/portgroups.tcl" "${port}" "${portvariants}"); do if [ "$portgroup" = "obsolete-1.0" ]; then exclude=1 exclude_reasons+=("it includes the obsolete 1.0 PortGroup") @@ -76,7 +76,7 @@ print-subports() { fi if [[ $exclude -eq 0 && ("${os_version}" = "10.6" || "${os_version}" = "10.5") ]]; then - supported_archs=$("${tclsh}" "${thisdir}/tools/supported-archs.tcl" "${port}") + supported_archs=$("${tclsh}" "${thisdir}/tools/supported-archs.tcl" "${port}" "${portvariants}") if [[ -n "${supported_archs}" ]]; then is_64bit_capable="$(sysctl -n hw.cpu64bit_capable)" if [[ "${os_version}" = "10.6" && "${is_64bit_capable}" = "0" && ! ("${supported_archs}" == *"x86_64"* && "${supported_archs}" == *"i386"*) ]]; then @@ -90,7 +90,11 @@ print-subports() { fi if [ $exclude -eq 0 ]; then - echo "$port" + if [ -n "${portvariants}" ]; then + echo "${port}@${portvariants}" + else + echo "$port" + fi else if [ ${#exclude_reasons[@]} -eq 1 ]; then echo >&2 "Excluding '${port}' because ${exclude_reasons[0]}."