list-storage

#!/bin/bash
# list storage by school
# Ruoshi Sun
# 2020-10-19

# parse argument
case $1 in
    CLAS|SDS|SEAS|SOM|EHD)
        ARG=$1
        PRINT_ALL=false
        ;;
    all) 
        PRINT_ALL=true
        ;;
    *)
        echo "Usage: `basename $0` CLAS|SDS|SEAS|SOM|EHD|all"
        exit 1
        ;;
esac

DTNDIR=/standard/rc-staff/dtn
STANDARD=$DTNDIR/ceph-list
GPFS=$DTNDIR/gpfs-list
#ICS=$DTNDIR/ivy-list
QUMULO=$DTNDIR/qumulo-list

for i in $STANDARD $GPFS $ICS $QUMULO; do
    if [ ! -e $i ]; then
        echo "Storage list $i not found"
        exit 1
    fi
done

function parse_ldap {
    FIELD=$1
    FILE=$2
    if [ "$FIELD" = "displayName" ]; then
        sed -n "s/^$FIELD: //p" $FILE | tr ' ' '_'
    elif [ "$FIELD" = "uvaDisplayDepartment" ]; then
        sed -n "s/^$FIELD: [^:]*://p" $FILE | tail -1 | tr ' ' '_'
    else
        echo "Unknown field $FIELD"
        exit 1
    fi
}

function get_ceph {
    getfattr --only-values -n $2 $1 2>/dev/null|awk 'END{print $1/1024**4}'
}

total_size=0
total_used=0

TMP=$(mktemp)
#echo $TMP
TMPLDAP=$(mktemp)
LOG=storage.log

IN=($STANDARD $GPFS $ICS $QUMULO)
TYPES=(standard project ics qumulo)
DIRS=(standard project "" project)

printf "%-6s %-40s %-8s %-35s %-25s %-8s %-10s %-10s\n" School Department Username Name Volume Type "Size (TB)" "Used (TB)" >$LOG
printf "%0.s-" {1..148} >>$LOG
echo >>$LOG

for i in ${!IN[@]}; do
    {
        TYPE=${TYPES[$i]}
        DIR=${DIRS[$i]}
        total=$(wc -l < ${IN[$i]})
        total=$((total-1))
        count=1
        read # skip first line
        while read -r line; do
            >&2 echo -ne "Reading ${IN[$i]} ($count/$total)\r"

            # could have multiple owners
            USERNAME=`echo $line|awk -F: '{print $8}'|awk '{print $1}'`
            VOLUME=`echo $line|awk -F: '{print $1}'`

            if [ ! -d /$DIR/$VOLUME ]; then
                count=$((count+1))
                continue
            fi

            ldapsearch -x -LLL -h ldap.virginia.edu -b "o=University of Virginia,c=US" uid=$USERNAME >$TMPLDAP
            NAME=$(parse_ldap displayName $TMPLDAP)
            SCHOOL_DEPT=$(parse_ldap uvaDisplayDepartment $TMPLDAP)

            # deal with school & department
            case "${SCHOOL_DEPT%%-*}" in
                "AS"|"Arts_&_Sciences_Graduate")
                    SCHOOL=CLAS ;;
                "DS"|"Data_Science"|"Data_Science_Graduate")
                    SCHOOL=SDS ;;
                "ED"|"Education_Graduate")
                    SCHOOL=EHD ;;
                "EN"|"Engineering_Graduate")
                    SCHOOL=SEAS ;;
                "MD"|"Medicine")
                    SCHOOL=SOM ;;
                *)
                    #>&2 echo "$USERNAME $SCHOOL_DEPT"
                    if [ "$PRINT_ALL" = "false" ]; then
                        count=$((count+1))
                        continue
                    fi
                    SCHOOL=Other
                    ;;
            esac

            if [ "$PRINT_ALL" = "false" ]; then
                if [ ! "$SCHOOL" = "$ARG" ]; then
                    count=$((count+1))
                    continue
                fi
            fi

            case $TYPE in
                project)
                    read -r USED SIZE < <(/usr/lpp/mmfs/bin/mmlsquota -j $VOLUME --block-size G tardis|awk 'END{print $3/1024,$4/1024}')
                    ;;
                standard)
                    SIZE=$(get_ceph /$DIR/$VOLUME ceph.quota.max_bytes)
                    USED=$(get_ceph /$DIR/$VOLUME ceph.dir.rbytes)
                    ;;
                ics)
                    SIZE=$(echo $line|awk -F: '{print $4}'|awk '{print $1}')
                    # unknown
                    USED="0"
                    ;;
                qumulo)
                    read -r SIZE USED < <(df /$DIR/$VOLUME | awk 'END{print $2/1024**3,$3/1024**3}')
                    ;;
                *)
                    echo "Unknown type"
                    exit 1
                    ;;
            esac

            if [ -z "$USERNAME" ]; then
                USERNAME="-"
            fi
            if [ -z "$NAME" ]; then
                NAME="-"
            fi
            printf "%-6s %-40s %-8s %-35s %-25s %-8s %10.4f %10.4f\n" "$SCHOOL" "$SCHOOL_DEPT" "$USERNAME" "$NAME" "$VOLUME" "$TYPE" "$SIZE" "$USED"

            count=$((count+1))
        done
        >&2 echo
    } < ${IN[$i]} >> $TMP
done
rm $TMPLDAP

sort -k1 $TMP >>$LOG
echo >>$LOG

#awk '{
#    size+=$6
#    used+=$7
#} END {
#    printf "Total size: %.4f TB\n", size
#    printf "Total used: %.4f TB (%.4f%%)\n", used, used/size*100
#}' $TMP >>$LOG

rm $TMP
echo "Done. Saved to $LOG"