diff --git a/Management-Utilities/fsx-ontap-aws-cli-scripts/list_fsxn_filesystems b/Management-Utilities/fsx-ontap-aws-cli-scripts/list_fsxn_filesystems index 7242d5f..422f141 100755 --- a/Management-Utilities/fsx-ontap-aws-cli-scripts/list_fsxn_filesystems +++ b/Management-Utilities/fsx-ontap-aws-cli-scripts/list_fsxn_filesystems @@ -25,6 +25,8 @@ # o Subnet ID - optional # o ARN - optional # o Backup - The Backup retention period. +# o Capacity - Optional +# o Used Capacity - Optional # # In the case of the Management IP and Subnet ID, it will only show the first # one defined. Based on the potential output from the API call, there could @@ -47,10 +49,10 @@ Usage $(basename $0) [-r region] [-a] [-n] [-c] [-s] [-b] [-i fileSystemId] [-f -c means to display a hierarchical view of each filesystem including svms and volumes. -i allows you to limit the display to the file system with the id provided. -f allows you to limit the display to the file system with the name provided. - -b means to display the backup retenion period. Not compoatible with the -x or -c options. + -b means to display the backup retenion period. Not compatible with the -x or -c options. -s means to display the current status of a volume. Only relative with the -c option. -n means to show the AWS ARN for the file system. Not compatible with -x or -c options. - -x means include additional information: vpc, subnet, Size, Deployement Type, Throughput, provisioned IOPS. + -x means include additional information: vpc, subnet, Size, Deployement Type, Throughput, provisioned IOPS, usage. EOF exit 1 } @@ -62,9 +64,12 @@ tmpout=/tmp/list_fss-out.$$ fileSystemsFile=/tmp/list_fss-fss.$$ svmsFile=/tmp/list_fss-svms.$$ volumesFile=/tmp/list_fss-volumes.$$ -trap 'rm -f $tmpout $fileSystemsFile $svmsFile $volumesFile' exit +cloudwatchOutputFile=/tmp/list_fss-cloudwatch-output.$$ +cloudwatchQueryFile=/tmp/list_fss-cloudwatch-query.$$ +fsCapacityFile=/tmp/list_fss-fscapacity.$$ +trap 'rm -f $tmpout $fileSystemsFile $svmsFile $volumesFile $cloudwatchOutputFile $cloudwatchQueryFile $fsCapacityFile' exit # -# Ensure all the required ultilities are installed. +# Ensure all the required utilities are installed. if which aws jq > /dev/null 2>&1; then : else @@ -253,7 +258,52 @@ for region in ${regions[*]}; do largestName=$(awk -F, 'BEGIN {max=0} {if(length($3) > max){max=length($3)}} END {print max}' < $tmpout) nameFieldWidth=$((largestName + 4)) if [ "$includeExtraInfo" == "true" ]; then - awk -F, -v region=$region 'BEGIN {first=1; formatStr="%12s %23s %'$nameFieldWidth's %14s %15s %22s %25s %6s %12s %11s %6s\n"}; {if(first) {printf "\n"; printf formatStr, "Region", "FileSystem ID", "Name", "Status", "Management IP", "VPC ID", "Subnet ID", "Size", "Deployment", "Throughput", "Iops"; first=0}; printf formatStr, region, $1, "\"" $3 "\"", $4, $5, $6, $7, $(12), $9, $(11), $(10)}' < $tmpout + fsxIds=$(awk -F, '{print $1}' < $tmpout) + echo "[" > $cloudwatchQueryFile + first=true + for fsid in $fsxIds; do + fsid2=$(echo $fsid | sed -e 's/-/_/g') + if [ "$first" == "true" ]; then + first=false + else + echo "," >> $cloudwatchQueryFile + fi + cat <> $cloudwatchQueryFile + { + "Id": "${fsid2}_StorageUsed", + "MetricStat": { + "Metric": { + "Namespace": "AWS/FSx", + "MetricName": "StorageUsed", + "Dimensions": [ + { + "Name": "FileSystemId", + "Value": "${fsid}" + }, + { + "Name": "StorageTier", + "Value": "SSD" + }, + { + "Name": "DataType", + "Value": "All" + }] + }, + "Period": 300, + "Stat": "Average" + } + } +EOF + done + echo "]" >> $cloudwatchQueryFile + aws cloudwatch get-metric-data --region=$region --metric-data-queries file://$cloudwatchQueryFile --start-time=$(date -u -d '5 minutes ago' +"%Y-%m-%dT%H:%M:%SZ") --end-time=$(date -u +"%Y-%m-%dT%H:%M:%SZ") --output=json > $cloudwatchOutputFile 2>&1 + for fsid in $fsxIds; do + fsid2=$(echo $fsid | sed -e 's/-/_/g') + storageUsed=$(jq -r '.MetricDataResults[] | if(.Id == "'${fsid2}_StorageUsed'") then .Values[0] // 0 else empty end' $cloudwatchOutputFile) + echo "$fsid,$storageUsed" >> $fsCapacityFile + done + + awk -F, -v region=$region -v fsCapacityFile=$fsCapacityFile 'BEGIN {first=1; formatStr="%12s %23s %'$nameFieldWidth's %14s %15s %22s %25s %10s %10s %12s %11s %6s\n"; while (getline < fsCapacityFile) {fsidCapacities[$1]=sprintf("%10.0f",$2/1024/1024/1024)}}; {if(first) {printf "\n"; printf formatStr, "Region", "FileSystem ID", "Name", "Status", "Management IP", "VPC ID", "Subnet ID", "Size(GiB)", "Used(GiB)", "Deployment", "Throughput", "Iops"; first=0}; printf formatStr, region, $1, "\"" $3 "\"", $4, $5, $6, $7, $(12), fsidCapacities[$1], $9, $(11), $(10)}' < $tmpout else if [ "$showARN" == "true" ]; then awk -F, -v region=$region 'BEGIN {first=1; formatStr="%12s %23s %70s %'$nameFieldWidth's %14s %15s\n"}; {if(first) {printf "\n"; printf formatStr, "Region", "FileSystem ID", "ARN", "Name", "Status", "Management IP"; first=0}; printf formatStr, region, $1, $2, "\"" $3 "\"", $4, $5}' < $tmpout diff --git a/Management-Utilities/fsx-ontap-aws-cli-scripts/list_fsxn_volumes b/Management-Utilities/fsx-ontap-aws-cli-scripts/list_fsxn_volumes index 405fd85..2ddf394 100755 --- a/Management-Utilities/fsx-ontap-aws-cli-scripts/list_fsxn_volumes +++ b/Management-Utilities/fsx-ontap-aws-cli-scripts/list_fsxn_volumes @@ -17,10 +17,14 @@ # It will list: # o Region # o File System ID -# o File System Name - optional +# o File System Name - optional. Comes with -n option +# o Storage Virtual Machine ID +# o Storage Virtual Machine Name - optional. Comes with -n option # o Volume ID -# o Volume Name # o Volume Status +# o Volume Size in MB - optional. Comes with -x option +# o Volume Used in MB - optional. Comes with -x option +# o Volume Name # ################################################################################ @@ -29,14 +33,16 @@ ################################################################################ usage () { cat 1>&2 < /dev/null 2>&1; then @@ -66,7 +75,9 @@ excludeRoot=false filter="" fsid="" fileSystemName="" -while getopts "hr:af:i:nos:" option; do +extraInfo=false +volumePattern="*" +while getopts "hr:af:i:nos:x" option; do case "$option" in r) region="$OPTARG" ;; @@ -83,13 +94,24 @@ while getopts "hr:af:i:nos:" option; do ;; s) svmID="$OPTARG" ;; + x) extraInfo=true + ;; *) usage ;; esac done +shift $((OPTIND-1)) +if [ ! -z "$1" ]; then + volumePattern="$1" +fi -if [ ! -z "$fileSystenName" -a ! -z "$fsid" ]; then - echo "Error, you can't provide both -f and -n options." 1>&2 +if [ ! -z "$fileSystemName" -a ! -z "$fsid" ]; then + echo "Error, you can't provide both -f and -i options." 1>&2 + exit 1 +fi + +if [ "$extraInfo" != "false" -a "$includeFsName" != "false" ]; then + echo "Error, you can't provide both -n and -x options." 1>&2 exit 1 fi @@ -128,33 +150,80 @@ if [ "$allRegions" = "true" ]; then else regions=($region) fi + +jqFields='\(.FileSystemId),\(.Name),\(.VolumeId),\(.Lifecycle),\(.OntapConfiguration.SizeInMegabytes),\(.OntapConfiguration.StorageVirtualMachineId)' # # Loop on all the regions. -fields='\(.FileSystemId),\(.Name),\(.VolumeId),\(.Lifecycle),\(.OntapConfiguration.StorageVirtualMachineId)' for region in ${regions[*]}; do # # Check that the fsx service is supported in thie region if [ ! -z "$(getent hosts fsx.$region.amazonaws.com)" ]; then if [ -z "$svmID" ]; then if [ "$excludeRoot" != "true" ]; then - aws fsx describe-volumes $filter --region=$region --output=json | jq -r '.Volumes[] | "'$fields'"' | sort > $tmpout + aws fsx describe-volumes $filter --region=$region --output=json | jq -r '.Volumes[] | "'$jqFields'"' | egrep ",$volumePattern," | sort > $volumeList else - aws fsx describe-volumes $filter --region=$region --output=json | jq -r '.Volumes[] | if(.OntapConfiguration.StorageVirtualMachineRoot | not) then "'$fields'" else empty end' | sort > $tmpout + aws fsx describe-volumes $filter --region=$region --output=json | jq -r '.Volumes[] | if(.OntapConfiguration.StorageVirtualMachineRoot | not) then "'$jqFields'" else empty end' | egrep ",$volumePattern," | sort > $volumeList fi else if [ "$excludeRoot" != "true" ]; then - aws fsx describe-volumes $filter --region=$region --output=json | jq -r '.Volumes[] | if(.OntapConfiguration.StorageVirtualMachineId == "'$svmID'") then "'$fields'" else empty end' | sort > $tmpout + aws fsx describe-volumes $filter --region=$region --output=json | jq -r '.Volumes[] | if(.OntapConfiguration.StorageVirtualMachineId == "'$svmID'") then "'$jqFields'" else empty end' | egrep ",$volumePattern," | sort > $volumeList else - aws fsx describe-volumes $filter --region=$region --output=json | jq -r '.Volumes[] | if(.OntapConfiguration.StorageVirtualMachineId == "'$svmID'" and (.OntapConfiguration.StorageVirtualMachineRoot | not)) then "'$fields'" else empty end' | sort > $tmpout + aws fsx describe-volumes $filter --region=$region --output=json | jq -r '.Volumes[] | if(.OntapConfiguration.StorageVirtualMachineId == "'$svmID'" and (.OntapConfiguration.StorageVirtualMachineRoot | not)) then "'$jqFields'" else empty end' | egrep ",$volumePattern," | sort > $volumeList fi fi if [ $includeFsName == "true" ]; then - aws fsx describe-file-systems --region=$region --output=json | jq -r '.FileSystems[] | .FileSystemId + "," + (.Tags[] | select(.Key == "Name") .Value)' | fgrep "$fileSystemName" > $tmpout2 - aws fsx describe-storage-virtual-machines --region=$region --output=json | jq -r '.StorageVirtualMachines[] | "\(.StorageVirtualMachineId),\(.Name)"' > $tmpout3 - awk -F, -v region=$region 'BEGIN {first=1; maxFsNameLen=0; while(getline < "'$tmpout2'") {fss[$1]=$2; if(length($2) > maxFsNameLen) {maxFsNameLen=length($2)}}; maxFsNameLen +=2; maxSvmNameLen=0; while(getline < "'$tmpout3'") {if(length($2) > maxSvmNameLen){maxSvmNameLen=length($2)}; svmNames[$1]=$2}; maxSvmNameLen+=2; formatStr="%12s %21s%-"maxFsNameLen"s %22s%-"maxSvmNameLen"s %24s %10s %s\n"}; {if(first) {printf "\n"; printf formatStr, "Region", "FileSystem ID", "(Name)", "Vserver", "(Name)", "Volume ID", "State", "Volume Name"; first=0}; fsName="("fss[$1]")"; svmName="("svmNames[$5]")"; printf formatStr, region, $1, fsName, $5, svmName, $3, $4, $2}' < $tmpout + aws fsx describe-file-systems --region=$region --output=json | jq -r '.FileSystems[] | .FileSystemId + "," + (.Tags[] | select(.Key == "Name") .Value)' > $fsNames + aws fsx describe-storage-virtual-machines --region=$region --output=json | jq -r '.StorageVirtualMachines[] | "\(.StorageVirtualMachineId),\(.Name)"' > $svmNames + awk -F, -v region=$region 'BEGIN {first=1; maxFsNameLen=0; maxSvmNameLen=0; while(getline < "'$fsNames'") {fss[$1]=$2; if(length($2) > maxFsNameLen) {maxFsNameLen=length($2)}}; maxFsNameLen +=2; while(getline < "'$svmNames'") {svm[$1]=$2; if(length($2) > maxSvmNameLen) {maxSvmNameLen=length($2)}}; maxSvmNameLen +=2; formatStr="%12s %21s%-"maxFsNameLen"s %21s-%-"maxSvmNameLen"s %24s %10s %s\n"}; {if(first) {printf "\n"; printf formatStr, "Region", "FileSystem ID", "(Name)", "SVM", "(Name)", "Volume ID", "State", "Volume Name"; first=0}; fsName="("fss[$1]")"; svmName="("svm[$6]")"; printf formatStr, region, $1, fsName, $6, svmName, $3, $4, $2}' < $volumeList else - awk -F, -v region=$region 'BEGIN {first=1; formatStr="%12s %21s %22s %24s %10s %s\n"}; {if(first) {printf "\n"; printf formatStr, "Region", "FileSystem ID", "Vserver", "Volume ID", "State", "Volume Name"; first=0}; printf formatStr, region, $1, $5, $3, $4, $2}' < $tmpout + if [ $extraInfo == "true" ]; then + volIds=$(awk -F, '{print $3}' < $volumeList) + if [ ! -z "$volIds" ]; then + echo "[" > $cloudwatchQueryFile + first=true + for volId in $volIds; do + volId2=$(echo $volId | sed -e 's/-/_/g') + if [ "$first" = "true" ]; then + first=false + else + echo "," >> $cloudwatchQueryFile + fi + cat <> $cloudwatchQueryFile + { + "Id": "m$volId2", + "MetricStat": { + "Metric": { + "Namespace": "AWS/FSx", + "MetricName": "StorageUsed", + "Dimensions": [ + { + "Name": "VolumeId", + "Value": "$volId" + }, + { + "Name": "FileSystemId", + "Value": "$(awk -F, -v volId=$volId '$3 == volId {print $1}' < $volumeList)" + }] + }, + "Period": 300, + "Stat": "Average" + } + } +EOF + done + echo "]" >> $cloudwatchQueryFile + aws cloudwatch get-metric-data --region $region --metric-data-queries file://$cloudwatchQueryFile --start-time=$(date -u -d '5 minutes ago' +"%Y-%m-%dT%H:%M:%SZ") --end-time=$(date -u +"%Y-%m-%dT%H:%M:%SZ") --output=json > $cloudwatchOutputFile 2>&1 + for volId in $volIds; do + volId2=$(echo $volId | sed -e 's/-/_/g') + capacity=$(jq -r '.MetricDataResults[] | select(.Id == "m'$volId2'") | .Values[0] // 0' < $cloudwatchOutputFile) + echo "$volId,$capacity" >> $volCapacityFile + done + awk -F, -v region=$region -v volCapacityFile=$volCapacityFile 'BEGIN {first=1; formatStr="%12s %21s %21s %24s %10s %16s %15s %s\n"; while (getline < volCapacityFile) {volidCapacities[$1]=sprintf("%10.0f",$2/1024/1024)}}; {if(first) {printf "\n"; printf formatStr, "Region", "FileSystem ID", "SVM", "Volume ID", "State", "Volume Size (MB)", "Used (MB)", "Volume Name"; first=0}; printf formatStr, region, $1, $6, $3, $4, $5, volidCapacities[$3], $2}' < $volumeList + fi + else + awk -F, -v region=$region 'BEGIN {first=1; formatStr="%12s %21s %21s %24s %10s %s\n"}; {if(first) {printf "\n"; printf formatStr, "Region", "FileSystem ID", "SVM", "Volume ID", "State", "Volume Name"; first=0}; printf formatStr, region, $1, $6, $3, $4, $2}' < $volumeList + fi fi else if [ $allRegions != "true" ]; then