Merge pull request #265 from NetApp/add_s3_endpoint_support

Modified delete_fsxn_volume to handle deleting a volume that has s3 access points attached to it.

Author: nichollri

Management-Utilities/fsx-ontap-aws-cli-scripts/delete_fsxn_filesystem

@@ -176,7 +176,7 @@ numSvms=$(jq '.StorageVirtualMachines | length' $svmsFile)
 if [ $numVolumes -gt 0 -o $numSvms -gt 0 ]; then
   echo "Here are the current contents of the '$fileSystemName'($fsid) file system you have indicated you want to delete:"
   displayFileSystemContents $fsid
-  read -p "Are you sure you want to delete this file system, with all the above volumes (yes/no)? " response
+  read -p "Are you sure you want to delete this file system, with all the volumes listed above and any attached s3 Access Points (yes/no)? " response
   if [ "$response" != "yes" ]; then
     echo "Aborted."
     exit 1

Management-Utilities/fsx-ontap-aws-cli-scripts/delete_fsxn_volume

@@ -30,6 +30,48 @@ EOF
   exit 1
 }
 
+################################################################################
+#  This function waits for the an s3 access point attached to a volume
+#  to be deleted.
+################################################################################
+waitForS3ApDelete () {
+
+  local s3ApName=$1
+  local volumeId=$2
+  local MaxIterations=60
+  local SleepTime=5
+  #
+  # Wait for the s3ap to be deleted.
+  i=0
+  while [ $i -lt $MaxIterations ]; do
+    aws fsx describe-s3-access-point-attachments --names $s3ApName --output=json --region=$region > $tmpout 2>&1
+    if [ $? -eq 0 ]; then
+      status=$(jq -r .S3AccessPointAttachments[0].Lifecycle $tmpout 2> /dev/null)
+      if [ "$status" != "DELETING" -a "$status" != "PENDING" ]; then
+        printf "\nError, failed to delete volume S3 Access Point $s3ApName attached to volume ID '$volumeId'. Status = ${status}.\n" 1>&2
+        reason="$(jq -r '.S3AccessPointAttachments[0].LifecycleTransitionReason.Message' $tmpout 2> /dev/null)"
+        if [ ! -z "$reason" ]; then
+          echo "Reason: $reason" 1>&2
+        else
+          cat $tmpout 1>&2
+        fi
+        return 1
+      fi
+    else
+      # Assume if it failed, it is because the s3 AP was deleted and doesn't exist anymore.
+      break
+    fi
+    [ $quiet != "true" ] && printf "."
+    sleep $SleepTime
+    let i+=1
+  done
+  if [ $i -ge $MaxIterations ]; then
+    printf "\nFailed to delete volume with volume ID of '$volumeId'. Taking too long.\n" 1>&2
+    return 1
+  fi
+  return 0
+}
+
 ################################################################################
 #  This function waits for the volume to be deleted.
 ################################################################################
@@ -113,7 +155,21 @@ if [ -z "$volumeId" ]; then
   usage
 fi
 
-aws fsx delete-volume --volume-id $volumeId --region=$region --output=json --ontap-configuration '{"SkipFinalBackup": '$skipBackup'}' > $tmpout 2>&1
+aws fsx describe-s3-access-point-attachments --filter "Name=volume-id,Values=$volumeId" --region=$region --output=json > $tmpout 2>&1
+if [ "$(jq -r '.S3AccessPointAttachments | length' $tmpout 2>&1)" != "0" ]; then
+  while read s3apName; do
+    [ $quiet != "true" ] && printf "S3 Access Point '$s3apName' is being deleted." 1>&2
+    aws fsx detach-and-delete-s3-access-point --name $s3apName --region=$region --output=json > $tmpout 2>&1
+    if waitForS3ApDelete $s3apName $volumeId; then
+      :
+    else
+      exit 1
+    fi
+    [ $quiet != "true" ] && printf "\n"
+  done < <(jq -r '.S3AccessPointAttachments[] | .S3AccessPoint.ResourceARN | split("/")[1]' $tmpout 2>/dev/null)
+fi
+
+aws fsx delete-volume --volume-id $volumeId --region=$region --output=json --ontap-configuration '{"SkipFinalBackup": '$skipBackup'}' --output=json > $tmpout 2>&1
 
 if [ $? != "0" ]; then
   echo "Failed to delete volume." 1>&2

Management-Utilities/fsx-ontap-aws-cli-scripts/list_aws_subnets

@@ -51,8 +51,8 @@ for region in $regions; do
     printf "\nRegion: $region\n"
   fi
   if [ -z "$vpcId" ]; then
-    aws ec2 describe-subnets --region=$region | jq -r '.Subnets[] | .VpcId + "," + .SubnetId + "," + .CidrBlock + "," + (if(has("Tags")) then first(.Tags[] | select(.Key == "Name").Value) // "" else "" end)' | awk -F, 'BEGIN {formatStr="%21s  %24s  %18s  %s\n"; printf(formatStr, "VPC Id", "Subnet ID", "CIDR", "Name")} {printf(formatStr , $1, $2, $3, $4)}'
+    aws ec2 describe-subnets --region=$region --output=json | jq -r '.Subnets[] | .VpcId + "," + .SubnetId + "," + .CidrBlock + "," + (if(has("Tags")) then first(.Tags[] | select(.Key == "Name").Value) // "" else "" end)' | awk -F, 'BEGIN {formatStr="%21s  %24s  %18s  %s\n"; printf(formatStr, "VPC Id", "Subnet ID", "CIDR", "Name")} {printf(formatStr , $1, $2, $3, $4)}'
   else
-    aws ec2 describe-subnets --region=$region --filters '[{"Name": "vpc-id", "Values": ["'$vpcId'"]}]' | jq -r '.Subnets[] | .VpcId + "," + .SubnetId + "," + .CidrBlock + "," + (if(has("Tags")) then first(.Tags[] | select(.Key == "Name").Value) // "" else "" end)' | awk -F, 'BEGIN {formatStr="%21s  %24s  %18s  %s\n"; printf(formatStr, "VPC Id", "Subnet ID", "CIDR", "Name")} {printf(formatStr , $1, $2, $3, $4)}'
+    aws ec2 describe-subnets --region=$region --output=json --filters '[{"Name": "vpc-id", "Values": ["'$vpcId'"]}]' | jq -r '.Subnets[] | .VpcId + "," + .SubnetId + "," + .CidrBlock + "," + (if(has("Tags")) then first(.Tags[] | select(.Key == "Name").Value) // "" else "" end)' | awk -F, 'BEGIN {formatStr="%21s  %24s  %18s  %s\n"; printf(formatStr, "VPC Id", "Subnet ID", "CIDR", "Name")} {printf(formatStr , $1, $2, $3, $4)}'
   fi
 done

Management-Utilities/fsx-ontap-aws-cli-scripts/list_aws_vpcs

@@ -59,7 +59,7 @@ vpcFormatStr="%21s  %19s  %s\n"
 for region in $regions; do
   [ "$quiet" != "True" ] && printf "\nRegion: $region\n"
   first=True
-  aws ec2 describe-vpcs --region=$region | jq -r '.Vpcs[] | .VpcId + " " + .CidrBlock + " " + (if (has("Tags")) then .Tags[] | (select(.Key == "Name") .Value) else "" end)' |  \
+  aws ec2 describe-vpcs --region=$region --output=json | jq -r '.Vpcs[] | .VpcId + " " + .CidrBlock + " " + (if (has("Tags")) then .Tags[] | (select(.Key == "Name") .Value) else "" end)' |  \
     while read vpcId cidr name; do
       if [ "$quiet" != "True" -a "$first" == "True" ]; then
         printf "\n$vpcFormatStr" "VPC IP" "CIDR" "Name"
@@ -69,7 +69,7 @@ for region in $regions; do
 
       if [ "$subnets" == "True" ]; then
         printf "\n\tSubnets:\n"
-        aws ec2 describe-subnets --region=$region --filters '[{"Name": "vpc-id", "Values": ["'$vpcId'"]}]' | jq -r '.Subnets[] | .VpcId + " " + .SubnetId + " " + .CidrBlock + " " + (if(has("Tags")) then first(.Tags[] | select(.Key == "Name").Value) // "" else "" end)' | awk 'BEGIN {formatStr="\t\t%24s  %18s  %s\n"; printf(formatStr, "Subnet ID", "CIDR", "Name")} {name=$4; for (i=5; i<=NF; i++) {name=name " " $(i)}; printf(formatStr , $2, $3, name)}'
+        aws ec2 describe-subnets --region=$region --output=json --filters '[{"Name": "vpc-id", "Values": ["'$vpcId'"]}]' | jq -r '.Subnets[] | .VpcId + " " + .SubnetId + " " + .CidrBlock + " " + (if(has("Tags")) then first(.Tags[] | select(.Key == "Name").Value) // "" else "" end)' | awk 'BEGIN {formatStr="\t\t%24s  %18s  %s\n"; printf(formatStr, "Subnet ID", "CIDR", "Name")} {name=$4; for (i=5; i<=NF; i++) {name=name " " $(i)}; printf(formatStr , $2, $3, name)}'
         first=True
       fi
   done

Management-Utilities/fsx-ontap-aws-cli-scripts/list_fsxn_filesystems

@@ -250,14 +250,16 @@ for region in ${regions[*]}; do
       jq -r '.[] | if(.FileSystemType == "ONTAP") then .FileSystemId + "," + .ResourceARN + "," + .Name + "," + .Lifecycle + "," + .ManagementIp + "," + .VpcId + "," + .SubnetId + "," + if(.AutomaticBackupRetentionDays == null) then "Dissabled" else (.AutomaticBackupRetentionDays | tostring) end + "," + .DeploymentType + "," + (.DiskIopsConfiguration | tostring) + "," + (.ThroughputCapacity | tostring) + "," + (.StorageCapacity | tostring) else empty end' $fileSystemsFile > $tmpout
     fi
 
+    largestName=$(awk -F, 'BEGIN {max=0} {if(length($3) > max){max=length($3)}} END {print max}' < $tmpout)
+    nameFieldWidth=$((largestName + 4))
     if [ "$includeExtraInfo" == "true" ]; then
-      awk -F, -v region=$region 'BEGIN {first=1; formatStr="%12s %23s %35s %10s %15s %22s %25s %6s %12s %11s %6s\n"}; {if(first) {printf "\n"; printf formatStr, "Region", "FileSystem ID", "Name", "Status", "Management IP", "VPC ID", "Subnet ID", "Size", "Deployment", "Throughput", "Iops"; first=0}; printf formatStr, region, $1, "\"" $3 "\"", $4, $5, $6, $7, $(12), $9, $(11), $(10)}' < $tmpout
+      awk -F, -v region=$region 'BEGIN {first=1; formatStr="%12s %23s %'$nameFieldWidth's %14s %15s %22s %25s %6s %12s %11s %6s\n"}; {if(first) {printf "\n"; printf formatStr, "Region", "FileSystem ID", "Name", "Status", "Management IP", "VPC ID", "Subnet ID", "Size", "Deployment", "Throughput", "Iops"; first=0}; printf formatStr, region, $1, "\"" $3 "\"", $4, $5, $6, $7, $(12), $9, $(11), $(10)}' < $tmpout
     else
       if [ "$showARN" == "true" ]; then
-        awk -F, -v region=$region 'BEGIN {first=1; formatStr="%12s %23s %70s %35s %10s %15s\n"}; {if(first) {printf "\n"; printf formatStr, "Region", "FileSystem ID", "ARN", "Name", "Status", "Management IP"; first=0}; printf formatStr, region, $1, $2, "\"" $3 "\"", $4, $5}' < $tmpout
+        awk -F, -v region=$region 'BEGIN {first=1; formatStr="%12s %23s %70s %'$nameFieldWidth's %14s %15s\n"}; {if(first) {printf "\n"; printf formatStr, "Region", "FileSystem ID", "ARN", "Name", "Status", "Management IP"; first=0}; printf formatStr, region, $1, $2, "\"" $3 "\"", $4, $5}' < $tmpout
       else
-        formatStr='%12s %23s %35s %10s %15s\n'
-        [ "$showBackup" == "true" ] && formatStr='%12s %23s %35s %10s %15s %17s\n'
+        formatStr='%12s %23s %'$nameFieldWidth's %14s %15s\n'
+        [ "$showBackup" == "true" ] && formatStr='%12s %23s %'$nameFieldWidth's %14s %15s %17s\n'
         awk -F, -v region=$region 'BEGIN {first=1; formatStr="'"${formatStr}"'"}; {if(first) {printf "\n"; printf formatStr, "Region", "FileSystem ID", "Name", "Status", "Management IP", "Backup Retention"; first=0}; printf formatStr, region, $1, "\"" $3 "\"", $4, $5, $8}' < $tmpout
       fi
     fi

Management-Utilities/fsx-ontap-aws-cli-scripts/list_fsxn_svms

@@ -126,7 +126,7 @@ for region in ${regions[*]}; do
   else
     filter="--filter Name=file-system-id,Values=$fileSystemID"
   fi
-  aws fsx describe-storage-virtual-machines --region=$region $filter | jq -r '.StorageVirtualMachines[] | "\(.FileSystemId),\(.StorageVirtualMachineId),\(.Endpoints.Nfs.IpAddresses[0]),\(.Name),\(.Lifecycle)"' | sort > $tmpout
+  aws fsx describe-storage-virtual-machines --output=json --region=$region $filter | jq -r '.StorageVirtualMachines[] | "\(.FileSystemId),\(.StorageVirtualMachineId),\(.Endpoints.Nfs.IpAddresses[0]),\(.Name),\(.Lifecycle)"' | sort > $tmpout
   if [ $includeIp == "true" ]; then
     ipFmt="%16s"
     ipHeader="IP"
@@ -136,7 +136,7 @@ for region in ${regions[*]}; do
   fi
 
   if [ $includeFsName == "true" ]; then
-    aws fsx describe-file-systems --region=$region | jq -r '.FileSystems[] | .FileSystemId + "," + (.Tags[] | select(.Key == "Name") .Value)' > $tmpout2
+    aws fsx describe-file-systems --output=json --region=$region | jq -r '.FileSystems[] | .FileSystemId + "," + (.Tags[] | select(.Key == "Name") .Value)' > $tmpout2
     awk -F, -v region=$region -v ipFmt=$ipFmt -v ipHeader=$ipHeader 'BEGIN {first=1; maxNameLen=0; while(getline < "'$tmpout2'") {fss[$1]=$2; if(length($2) > maxNameLen) {maxNameLen=length($2)}}; maxNameLen +=2; formatStr="%12s %20s%-"maxNameLen"s %23s "ipFmt" %13s %s\n"}; {if(first) {printf "\n"; printf formatStr, "Region", "FileSystem ID", "(Name)", "SVM ID", ipHeader, "State", "SVM Name"; first=0}; if(ipHeader != "IP") {ip=""} else {ip=$3}; name="("fss[$1]")"; printf formatStr, region, $1, name, $2, ip, $5, $4}' < $tmpout
   else
     awk -F, -v region=$region -v ipFmt=$ipFmt -v ipHeader=$ipHeader 'BEGIN {first=1; formatStr="%12s %23s %23s "ipFmt" %13s %s\n"}; {if(first) {printf "\n"; printf formatStr, "Region", "FileSystem ID", "SVM ID", ipHeader, "State", "SVM Name"; first=0}; if(ipHeader != "IP") {ip=""} else {ip=$3}; printf formatStr, region, $1, $2, ip, $5, $4}' < $tmpout