Merge pull request #8 from rlcee/sll_221213

updates

Author: rlcee

bin/dcacheFileInfo

@@ -23,6 +23,7 @@ mcs.mu2e.DS-cosmic-mix-cat.MDC2018i.001002_00000001.art
      -l print locality
         NEARLINE - only on tape, ONLINE- on disk
         ONLINE_AND_NEARLINE means both
+     -d print create date in epoch seconds
      -v verbose
      -t interpret argument as file name only, and lookup the tape location
      -p interpret argument as file name only, and lookup the persis. location
@@ -39,18 +40,22 @@ mcs.mu2e.DS-cosmic-mix-cat.MDC2018i.001002_00000001.art
 
 DOCRC=""
 DOLOC=""
+DODATE=""
 VERBOSE=""
 LTAPE=""
 LDISK=""
 
-while getopts cltpvh OPT; do
+while getopts cldtpvh OPT; do
     case $OPT in
         c)
             DOCRC=yes
             ;;
         l)
             DOLOC=yes
             ;;
+        d)
+            DODATE=yes
+            ;;
         t)
             LTAPE=yes
             ;;
@@ -94,24 +99,16 @@ if [[ -n "$LTAPE" || -n "$LDISK" ]]; then
         source /cvmfs/mu2e.opensciencegrid.org/setupmu2e-art.sh
         setup mu2efilename
     fi
-    if [ $LTAPE ]; then
-        FFS=$(echo -n $FFS | mu2eabsname_tape )
+    if [ "$LTAPE" ]; then
+        FFS=$(echo $FFS | mu2eabsname_tape )
     else
-        FFS=$(echo -n $FFS | mu2eabsname_disk )
+        FFS=$(echo $FFS | mu2eabsname_disk )
     fi
 fi
-#if [ "$STUB" != "/pnfs"  ]; then
-#    echo "ERROR - one pnfs filespec argument is required"
-#    exit 1
-#fi
 
 # canonical dCache filespec
 CFFS=$(echo $FFS | sed 's|/pnfs|/pnfs/fnal.gov/usr|' )
 
-
-
-#curl -L --capath /etc/grid-security/certificates --cert $FINV --cacert $FINV --key $FINV -X GET "https://fndca1.fnal.gov:3880/api/v1/namespace${CFFS}?checksum=true&locality=true"
-
 VS=" -s "
 [ $VERBOSE ] && VS=""
 
@@ -124,7 +121,7 @@ if [ $RC -ne 0 ]; then
     exit 1
 fi
 
-[ $VERBOSE ] && echo -n "$ANS"
+[ $VERBOSE ] && echo "$ANS"
 
 CRC=""
 if [ $DOCRC ]; then
@@ -133,9 +130,15 @@ fi
 
 LOC=""
 if [ $DOLOC ]; then
-  LOC=$( echo -n "$ANS" | grep fileLocality | tr -d '":,' | awk '{print$2}' )
+  LOC=$( echo -n "$ANS" | grep fileLocality | tr -d '":,' | awk '{print $2}' )
+fi
+
+DATE=""
+if [ $DODATE ]; then
+  DATEMS=$( echo -n "$ANS" | grep creationTime | tr -d '":,' | awk '{print $2}' )
+  DATE=$(($DATEMS/1000))
 fi
 
-[ ${CRC}${LOC} ] && echo $CRC $LOC
+[ ${CRC}${LOC} ] && echo $CRC $LOC $DATE
 
 exit 0

bin/samListLocations

@@ -55,19 +55,38 @@ elif [ "$1" == "-f" ]; then
     TMPF=$(mktemp)
 fi
 
+[ -z "$MU2E" ] && setup mu2e
+[ -z "$SETUP_SAM_WEB_CLIENT" ] && setup sam_web_client
+[ -z "$SETUP_DHTOOLS" ] && setup dhtools
+if ! voms-proxy-info > /dev/null 2>&1 ; then
+    vomsCert > /dev/null 2>&1
+fi
+
+TMPL=$(mktemp)
 
+samweb list-file-locations "${@}" | tr ":" " "  > $TMPL
 
-OLDIFS=${IFS}
-samweb list-file-locations "${@}" | while read line
+# loop over unique file names
+cat $TMPL | awk '{print $3}' | sort | uniq \
+| while read FILENAME
 do
-   IFS=$'\t'
-   read -a strarr <<< "$line"
-   FILEPATH=`echo ${strarr[0]} | cut -d':' -f2`
-   FILESPEC=${FILEPATH}/${strarr[1]}
-
-   LOC=""
-   if [ $DOLOC ]; then
-       LOC=$( dcacheFileInfo -l $FILESPEC )
+
+   # list of locations, dcache first
+   FSS=$(cat $TMPL | awk '{if($3=="'$FILENAME'") print $1" "$2}' | sort )
+
+   TAPESPEC=$(echo "$FSS" | awk '{if($1=="enstore") print $2"/'$FILENAME'"}')
+   DISKSPEC=$(echo "$FSS" | awk '{if($1=="dcache") print $2"/'$FILENAME'"}')
+
+   if [ "$DISKSPEC" ]; then
+       FILESPEC="$DISKSPEC"
+       LOC="ONLINE"
+   else
+       FILESPEC="$TAPESPEC"
+       LOC=""
+       if [ $DOLOC ]; then
+           # have to see if tape-backed is on disk
+           LOC=$( dcacheFileInfo -l $FILESPEC )
+       fi
    fi
 
    if [ $DODISK ]; then
@@ -83,11 +102,12 @@ do
    fi
 
 done
-IFS=${OLDIFS}
 
 if [ $DOFIRST ]; then
     cat $TMPF
     rm -f $TMPF
 fi
+rm -f $TMPL
+rm -f $TMPF
 
 exit 0

build/tarball.sh

@@ -0,0 +1,96 @@
+#!/bin/bash
+#
+# script to make a ups tarball
+# args: $1=version
+# tarball will be created in the cwd
+#
+VERSION="$1"
+if [ ! "$VERSION" ]; then
+    echo "ERROR - no required version argument"
+    exit 1
+fi
+
+OWD=$PWD
+SDIR=$(dirname $(readlink -f $BASH_SOURCE)  | sed 's|/build||' )
+
+PDIR=$(mktemp -d)
+cd $PDIR
+mkdir -p dhtools
+cd dhtools
+mkdir -p $VERSION
+cd $VERSION
+
+rsync --exclude "*~" --exclude "*__*"  \
+    -r $SDIR/bin $SDIR/python  .
+mkdir -p ups
+cd ups
+
+cat > dhtools.table <<EOL
+File    = table
+Product = dhtools
+
+#*************************************************
+# Starting Group definition
+
+Group:
+
+Flavor     = ANY
+Qualifiers = ""
+
+  Action = flavorSetup
+
+Flavor     = ANY
+Qualifiers = "grid"
+
+  Action = flavorSetup
+    sourceRequired(\${UPS_PROD_DIR}/bin/mgf_functions.sh,UPS_ENV)
+
+Common:
+  Action = setup
+    setupRequired( ifdhc  )
+    setupRequired( sam_web_client  )
+    prodDir()
+    setupEnv()
+    envSet(\${UPS_PROD_NAME_UC}_VERSION, $VERSION)
+    sourceRequired(\${UPS_PROD_DIR}/bin/dh_functions.sh,UPS_ENV)
+    pathPrepend(PATH,\${UPS_PROD_DIR}/bin)
+    pathPrepend(PYTHONPATH,\${UPS_PROD_DIR}/python)
+    exeActionRequired(flavorSetup)
+
+End:
+# End Group definition
+#*************************************************
+EOL
+
+# up to dhtools dir
+cd ../..
+
+mkdir -p ${VERSION}.version
+cd ${VERSION}.version
+
+cat > NULL <<EOL
+FILE = version
+PRODUCT = dhtools
+VERSION = $VERSION
+
+FLAVOR = NULL
+QUALIFIERS = ""
+  PROD_DIR = dhtools/$VERSION
+  UPS_DIR = ups
+  TABLE_FILE = dhtools.table
+
+FLAVOR = NULL
+QUALIFIERS = "grid"
+  PROD_DIR = dhtools/$VERSION
+  UPS_DIR = ups
+  TABLE_FILE = dhtools.table
+EOL
+
+cd ../..
+
+tar -cjf $OWD/dhtools-${VERSION}.bz2 dhtools/${VERSION} dhtools/${VERSION}.version
+rm -rf $PDIR/dhtools
+rmdir $PDIR
+
+cd $OWD
+exit 0