#!/bin/sh -ph
# *****************************************************************************
#   (C) Copyright 2000-2008 Hewlett-Packard Development Company, L.P.
#   HPVM: $Id: hpvmcollect 7744 2008-05-02 16:57:50Z spider $ - HP CONFIDENTIAL
# *****************************************************************************
# hpvmcollect:  collect information from a crash or dump
# ******************************************************

PATH=$PATH:/opt/hpvm/bin:/usr/sbin:/usr/ucb:/usr/bin:/sbin:/usr/contrib/bin
VER=4.0         # Current HPVM Version as of last change
CRASHDUMP=0     # Default is no crash dump collection
ZIP=1           # Default is to archive the directory into tar.gz
TARGETHOST=""
TARGETROOT=/crashes
RCP="scp -q -p"
WDIR=$(pwd)
FORCE=0
CRASHNUM=-1
CRASHDIR=
ONGUEST=""
CMD="hpvmcollect"
DATE=$(date '+%b.%d.%y_%H%M%S%Z')
DATEASIS=$(date)
GARBAGE=0
OSTYPE=$(uname -s)
OS_HPUX=HP-UX
OS_LINUX=Linux
KREV=$(uname -r)

# Variables used by the functions for handing logging of commands and general
# progress:

STEP_TAG=	# "  getting stuff " prefix shown to user
DOTS_COL=71	# column for padding to with dots (not counting trailing space)
DOTS_FILL=	# holds the dots fill
ST_GOOD=OK	# tag to print when a step succeeds
#ST_SKIP=N/A	# tag to print when a step is skipped
#ST_FAIL=FAIL	# tag to print when a step fails
ST_SKIP=NO	# Restore nonsensical values for these, since they're in the
ST_FAIL=N/A	#  manpage.
CMD_LOG=	# log file for current cmd_to_log() calls
TAG_TEE=	# non-blank for tag_start() / tag_finish() to "tee -a $RLOG"
                #   (needed to keep DOTS_FILL assignments in this process)
opt=		# make visible for dup() calls

# Variables for "option-seen" tracking:
bFLAG=
cFLAG=
dFLAG=
fFLAG=
gFLAG=
lFLAG=
nFLAG=
pFLAG=
PFLAG=
rFLAG=
sFLAG=

# Shell settings required (just in case)
set -h
set +u

# Functions used to make guest/host linux/hpux decisions a little easier to
# read in-line:

hpux_only ()
{
    typeset cmd=$1
    shift
    case "$OSTYPE" in
    "$OS_HPUX")
	$cmd "$@"
	;;
    *)
	return 1
	;;
    esac
} # hpux_only()

hpux_11iv3_only ()
{
    typeset cmd=$1
    shift
    case "$OSTYPE/$KREV" in
    "$OS_HPUX"/?.11.3*)
	$cmd "$@"
	;;
    *)
	return 1
	;;
    esac
} # hpux_11iv3_only()

linux_only ()
{
    typeset cmd=$1
    shift
    case "$OSTYPE" in
    "$OS_LINUX")
	$cmd "$@"
	;;
    *)
	return 1
	;;
    esac
} # linux_only()

host_only ()
{
    typeset cmd=$1
    shift
    case "$WHERE" in
    "host")
	$cmd "$@"
	;;
    *)
	return 1
	;;
    esac
} # host_only()

guest_only ()
{
    typeset cmd=$1
    shift
    case "$WHERE" in
    "guest")
	$cmd "$@"
	;;
    *)
	return 1
	;;
    esac
} # guest_only()

# The above "*_only" routines quietly return false on non-match for the benefit
# of the next few "is_*" aliases, used in more complicated if/then processing.

alias is_host="host_only :"
alias is_guest="guest_only :"
alias is_hpux="hpux_only :"
alias is_hpux_11iv3="hpux_11iv3_only :"
alias is_linux="linux_only :"


# Compensate for Linux's gratuitously broken posix-shell emulation:
linux_only shopt -s xpg_echo extglob


banner ()
{
    # first arg is output file descriptor

    echo >&$1 ""
    echo >&$1 "  HPVM ${WHERE} crash/log collection tool version $VER"
} # banner()


tag_start ()
{
    # arg 1 is the (NOT space-padded) info for the step
    STEP_TAG="  $1 "
    DOTS_FILL=$(printf "%0.$((DOTS_COL-${#STEP_TAG}))d" 0 | tr 0 .)
    if [[ -n "$TAG_TEE" ]]
    then
	printf %s "$STEP_TAG" | tee -a $RLOG
    else
	printf %s "$STEP_TAG"
    fi
} # tag_start()

tag_finish ()
{
    # arg 1 is the ST_{SKIP|GOOD|FAIL} status to print
    if [[ -n "$TAG_TEE" ]]
    then
	echo "$DOTS_FILL $1" | tee -a $RLOG
    else
	echo "$DOTS_FILL $1"
    fi
} # tag_finish()

copy_file_finish ()
{
    # Args are:
    # $1 - file to copy
    # $2 - destination
    if [[ -f "$1" ]]
    then
        if cp -fp "$1" "$2" >/dev/null 2>&1
        then
            tag_finish $ST_GOOD
        else
            tag_finish $ST_FAIL
        fi
    else
        tag_finish $ST_SKIP
    fi
} # copy_file_finish()

copy_file_step ()
{
    # Args are:
    # $1 - tag for tag_start
    # $2 - file to copy
    # $3 - destination
    tag_start "$1"
    copy_file_finish "$2" "$3"
} # copy_file_step()

new_cmd_log ()
{
    # arg 1 is the new log file for our commands in this step
    CMD_LOG=$1
    touch $CMD_LOG	# make sure it exists
} # new_cmd_log()

cmd_to_log ()
{
    # args are command to capture
    typeset cmd=$1 cmdline=$*
    shift
    if [[ -s $CMD_LOG ]]
    then
	# vertical whitespace between commands, but not at start of file
	echo "" >> $CMD_LOG
    fi
    echo ">>> $cmdline\n" >> $CMD_LOG
    $cmd "$@" >> $CMD_LOG 2>&1
} # cmd_to_log()

print_usage ()
{
    # first arg is output file descriptor

    banner $1   # print banner to either stdout or stderr

    if is_host
    then
       echo >&$1 "  Usage: $CMD [-fgh] [-b #] [-d dir] [-c [-n #]]" \
                 "[-l | -s host [-r dir]]"
       echo >&$1 "                     {-P vm_name | -p vm_number}"
    else
       echo >&$1 "  Usage: $CMD [-fh] [-b #] [-d dir] [-c [-n #]]" \
                 "[-l | -s host [-r dir]]"
    fi

    echo >&$1 ""
    echo >&$1 "      -b : bug report no."
    echo >&$1 "      -c : collect crash dump"
    echo >&$1 "      -d : local target directory to store the archive to"
    echo >&$1 "      -f : force write (replacing current file or dir)"
    if is_host
    then
        echo >&$1 "      -g : garbage-collect (clean-up) guest debug dump data"
    fi
    echo >&$1 "      -l : leave the collection as is (do not archive)"
    echo >&$1 "      -n : specifying crash dump number, option with -c"
    echo >&$1 "      -r : specify remote directory other than /crashes" \
              "(with -s)"
    echo >&$1 "      -s : host to send the archive file to (using scp)"
    if is_host
    then
        echo >&$1 "      -P : vm_name"
        echo >&$1 "      -p : vm_number"
    fi
    echo >&$1 "      -h : help"
    echo >&$1 ""
} # print_usage()

usage ()
{
    print_usage 2   # print usage to stderr (come here due to error condition)
    exit 1
} # usage()

badcrash ()
{
    echo >&2 "$CMD: Invalid crash \"${CRASHNUM}\" directory."
    usage
} # badcrash()

dup ()
{
     echo >&2 "$CMD: A duplicate option was used: -$opt"
     usage
} # dup()

example()
{
    # print example (help) message to stdout

    host_only \
    echo "  eg) $CMD -d /tmp/hpvmdump/ -c -n 23 -P testGuest"
    guest_only \
    echo "  eg) $CMD -d /tmp/hpvmdump/ -c -n 23"
    echo ""
    echo "      -d to store the collection in /tmp/hpvmdump/"
    echo "      -c to collect crash dump"
    echo "      -n to collect /var/adm/crash/crash.23"
    host_only \
    echo "      -P to specify the name of the guest"
    echo ""

    exit 0
} # example()
 
# Check to see if superuser
RUNOK=$(id -u)
if [[ "$RUNOK" != "0" ]]
then
    echo >&2 "$CMD: Must be superuser to run"
    echo >&2 ""
    exit 1
fi

# Find out if running on host (pman) or guest: expecting output like ":host:"
# or ":guest:" or ":neither:"
WHERE=$(hpvminfo -M 2>&1 | grep "^:" | cut -d : -f 2)
case "$WHERE" in
guest|host) ;;
'') WHERE=guest ;;
*)
    # neither host nor guest
    echo >&2 "$CMD: HPVM not installed"
    usage
    ;;
esac

######################################
# option processing & error checking #
######################################

OPTERR=0
# Split options string from one single set used always to handling guest vs
# host separately.  That way we get more consistent error messages for
# host-only options.
OPTSTRING_COMMON=":b:cd:fhHln:r:s:"
OPTSTRING_GUEST=
OPTSTRING_HOST="gP:p:"
if is_guest
then
    OPTSTRING="$OPTSTRING_COMMON$OPTSTRING_GUEST"
else
    OPTSTRING="$OPTSTRING_COMMON$OPTSTRING_HOST"
fi
while getopts "$OPTSTRING" opt > /dev/null 2>&1
do
    case $opt in
        b)  [[ -n "$bFLAG" ]] && dup
            # bug #, use this for directory name
            BUGRPT="$OPTARG"
            case "$BUGRPT" in
            .|..|*/*|*\\*)
                echo >&2 "$CMD: Invalid name with -b option."
                usage
                ;;
            esac
            GDIR="bug_${BUGRPT}"
            bFLAG=1
            ;;
        d)  [[ -n "$dFLAG" ]] && dup
            WDIR="$OPTARG"      # specified working directory
            dFLAG=1
            ;;
        f)  [[ -n "$fFLAG" ]] && dup
            FORCE=1             # force removal of original archive dir.
            fFLAG=1
            ;;
        g)  [[ -n "$gFLAG" ]] && dup
            GARBAGE=1           # garbage collect (cleanup) guest debug dir
            gFLAG=1
            ;;
        l)  [[ -n "$lFLAG" ]] && dup
            ZIP=0               # (don't) archive the dir.
            lFLAG=1
            ;;
        c)  [[ -n "$cFLAG" ]] && dup
            CRASHDUMP=1         # collect crash dump
            cFLAG=1
            ;;
        n)  [[ -n "$nFLAG" ]] && dup
            CRASHNUM="$OPTARG"  # specified crash dump number
            # Rather than have a linux-only feature here, intercept an absolute
            # directory as a value early and just take it.
            if [[ -d "$CRASHNUM" && "$CRASHNUM" = /* ]]
            then
                CRASHDIR=$CRASHNUM
            else
                if is_hpux
                then
                    if [[ ! -d /var/adm/crash/crash.${CRASHNUM} ]]
                    then
                        badcrash
                    fi
                    CRASHDIR=/var/adm/crash/crash.$CRASHNUM
                else
                    # Need to try to distinguish LKCD vs diskdump *sigh*.  The
                    # reason we accept an absolute directory up above is
                    # because this guess is only a guess, and we need a way for
                    # the customer to override it.
                    if [[ -d /var/log/dump ]]
                    then
                        if [[ ! -d "/var/log/dump/$CRASHNUM" ]]
                        then
                            badcrash
                        fi
                        CRASHDIR=/var/log/dump/$CRASHNUM
                    else
                        # Apparently a diskdump user.  Try a couple of guesses,
                        # and hope a lot.
                        if [[ -d "/var/crash/$CRASHNUM" ]]
                        then
                            CRASHDIR=/var/crash/$CRASHNUM
                        elif [[ -d "/var/crash/127.0.0.1-$CRASHNUM" ]]
                        then
                            CRASHDIR=/var/crash/127.0.0.1-$CRASHNUM
                        else
                            badcrash
                        fi
                    fi
                fi
            fi
            nFLAG=1
            ;;
        r)  [[ -n "$rFLAG" ]] && dup
            TARGETROOT=$OPTARG
            rFLAG=1
            ;;
        s)  [[ -n "$sFLAG" ]] && dup
            ZIP=1               # transfer the archive to the target host
            TARGETHOST="$OPTARG"
            sFLAG=1
            ;;
        p)  [[ -n "$pFLAG$PFLAG" ]] && dup
            GUESTNUM="$OPTARG"  # vm number 
            # Using -D here in case of hung guest
            hpvmstatus -D -p "${GUESTNUM}" > /dev/null 2>&1 || \
            {
                echo >&2 "$CMD: Invalid vm number ${GUESTNUM}."
                usage
            }
            # Using -D here in case of hung guest
            GUEST=$(hpvmstatus -D -M -p "${GUESTNUM}" | cut -f 1 -d :)
            pFLAG=1
            ;;
        P)  [[ -n "$PFLAG$pFLAG" ]] && dup
            GUEST="$OPTARG"     # name of the guest
            # check to see if the guest exists
            # Using -D here in case of hung guest
            hpvmstatus -D -P "${GUEST}" > /dev/null 2>&1 || \
            {
                echo >&2 "$CMD: Invalid vm \"${GUEST}\"."
                usage
            }
            PFLAG=1
            ;;
        H|h)
            print_usage 1       # help
            example
            ;;
        :)  # the first ":" in the optstring will set opt to ":" when 
            # no argument is given 
            echo >&2 "$CMD: Option -$OPTARG requires an argument."
            usage
            ;;
        \?) # getopts will set opt to "?" with unknown option string
            echo >&2 "$CMD: Unrecognized option: -$OPTARG."
            usage
            ;;
        *)  # Oddball corner cases like "+h" come here.
            echo >&2 "$CMD: Unrecognized option: $opt."
            usage
            ;;
    esac
done

# If no error from getopts, $# + 1 is equal to $OPTIND
if (($#+1 != OPTIND))
then
    echo >&2 "$CMD: Unexpected parameter specified."
    usage
fi

if is_host
then
    # If no P or p, check enviromental variable
    if [[ -z "$PFLAG$pFLAG" ]]
    then
        # If the HPVM_VMNAME env variable exists
        if [[ -n "$HPVM_VMNAME" ]]
        then
            # check to see if the guest exists
            # Using -D here in case of hung guest
            hpvmstatus -D -P "${HPVM_VMNAME}" > /dev/null 2>&1 || \
            {
                echo >&2 "$CMD: Invalid vm \"${HPVM_VMNAME}\"."
                usage
            }
            # Treat it as if the user specified it with -P
            GUEST=$HPVM_VMNAME
            PFLAG=1
        fi
    fi
fi

# check to see if -d option is valid
if [[ ! -d "$WDIR" ]]
then
    echo >&2 "$CMD: Invalid directory \"$WDIR\"."
    usage
elif [[ ! -w "$WDIR" ]]
then
    echo >&2 "$CMD: Write access denied on \"$WDIR\"."
    usage
fi

# Set GUEST name as hostname when running in a guest
if is_host
then
    # On  host, guest name is needed
    if [[ -z "$GUEST" ]]
    then
        echo >&2 "$CMD: The -p or -P option is required."
        usage
    fi
else
    GUEST=$(hostname)           # use hostname on guest 
    : "${GUEST:=localhost}"     # or 'localhost' if not set
fi

# Set archive name if no -b option
if [[ -z "$bFLAG" ]]
then
    GDIR=$GUEST\_$DATE
fi

# if -n option, then check to see if -c option is also specified
if [[ -n "$nFLAG" && -z "$cFLAG" ]]
then
    echo >&2 "$CMD: -n option requires -c option."
    usage
fi

# Ensure not trying to mix -l and -s options.
if [[ -n "$lFLAG" && -n "$TARGETHOST" ]]
then
    echo >&2 "$CMD: -l option incompatible with -s option."
    usage
fi

# The -r option requires the -s option.
if [[ -n "$rFLAG" && -z "$sFLAG" ]]
then
    echo >&2 "$CMD: -r option requires -s option."
    usage
fi

# Also, clean up naive values of the -r option.
if [[ -z "$TARGETROOT" ]]
then
    TARGETROOT=.
else
    TARGETROOT="${TARGETROOT%%+(/)}"
fi

# check to see if guest directory and config file exist
BROOT=/var/opt/hpvm/guests/$GUEST
CONFIG_FILE=$BROOT/vmm_config.current
if is_host
then
    if [[ ! -d "$BROOT" ]]
    then
        echo >&2 "$CMD: Invalid guest \"$GUEST\""
        usage
    fi
    if [[ ! -f "$CONFIG_FILE" ]]
    then
        echo >&2 "$CMD: Cannot find guest's config file."
        usage
    fi
fi


# mkdir parent working directory to store the collection to
WDIR="${WDIR%/}/hpvmcollect_archive"
[ ! -d $WDIR ] && {
    mkdir $WDIR >/dev/null 2>&1 || {
        echo >&2 "$CMD: Cannot create directory '$WDIR'."
        exit 1
    }
}

# user default archive directory as $GUESTNAME_$CURRENTDATE
cd $WDIR

# Create working directory. If exists and not -f, mv existing to $DATE appended
# directory, and then mkdir
if [[ ! -d $GDIR ]]
then
    mkdir $GDIR >/dev/null 2>&1 || {
        echo >&2 "$CMD: Cannot create directory '$GDIR'."
        exit 1
    }
else
    if [[ $FORCE -eq 1 ]]
    then
        rm -rf $GDIR >/dev/null 2>&1
    else
        echo "$CMD: Directory '$GDIR' already exists."
        echo "      Renamed with timestamp appended."
        mv -f $GDIR $GDIR\_$DATE >/dev/null 2>&1
    fi
    mkdir $GDIR >/dev/null 2>&1 || {
        echo >&2 "$CMD: Cannot create directory '$GDIR'."
        exit 1
    }
fi

# Create logs/ and config/ directory. mkdir images/ directory on host
cd $GDIR
if is_host
then
    mkdir images > /dev/null 2>&1 || {
        echo >&2 "$CMD: Create images directory."
        usage
    }
fi

mkdir logs > /dev/null 2>&1 || {
    echo >&2 "$CMD: Cannot create logs directory."
    usage
}

mkdir config > /dev/null 2>&1 || {
    echo >&2 "$CMD: Cannot create config directory."
    usage
}

# Check to see if crash dump exists (or find one)
if [[ -n "$cFLAG" && -z "$nFLAG" ]]
then
    if is_hpux
    then
        if [[ ! -f /var/adm/crash/bounds ]]
        then
            CRASHDIR=
        else
            CRASHNUM=$(($(</var/adm/crash/bounds)-1))
            CRASHDIR=/var/adm/crash/crash.$CRASHNUM
        fi
    else
        # More attempting to guess between LKCD and diskdump *sigh*
        if [[ -f /var/log/dump/bounds ]]
        then
            # Looks like LKCD.
            CRASHNUM=$(($(</var/log/dump/bounds)-1))
            CRASHDIR=/var/log/dump/$CRASHNUM
        else
            # Probably diskdump.  Different error for them if we can't guess.
            if [[ ! -d /var/crash ]]
            then
                CRASHDIR=
            else
                CRASHNUM=$(ls -1rd /var/crash/127.0.0.1-* 2>/dev/null | head -1)
                if [[ -n "$CRASHNUM" ]]
                then
                    CRASHDIR=$CRASHNUM
                else
                    CRASHDIR=
                fi
            fi
            if [[ -z "$CRASHDIR" ]]
            then
                echo >&2 "$CMD: Use -n option to specify crash dump directory."
                usage
            fi
        fi
    fi
    # Since the user did say to collect the dump, don't just blithely ignore
    # its non-existence.  Check.
    if [[ -z "$CRASHDIR" || ! -d "$CRASHDIR" ]]
    then
        echo >&2 "$CMD: Use -n option to specify crash dump number."
        usage
    fi
fi

# $RLOG will store exact copy of this hpvmcollect run log within the archive
RLOG=hpvmcollect.log
touch "$RLOG" || { 
        echo >&2 "$CMD: Cannot make log file."
        usage
}
################################################################
# Error checking and pre-processing complete. Do the real work #
################################################################
banner 1    # print banner to stdout

echo "  Gathering info for post-mortem analysis \c"
if is_host
then
    echo "of guest '$GUEST' on host $(hostname)"
else
    echo "on guest (hostname '$GUEST')"
fi
echo ""

# print banner to $RLOG with timestamp
echo "\n  HPVM $WHERE crash/log collection tool version $VER" > $RLOG
echo "  Gathering information for post-mortem analysis of guest '$GUEST'" >> $RLOG
echo "  Ran on ${DATEASIS}\n" >> $RLOG

TAG_TEE=1

# Save host's hpvm device database
#
# NOTE: if hpvm_mgmtdb does not exist, the $ST_SKIP message
# will display, but hpvm utilities called further down
# in hpvmcollect will create a copy.  This utility does
# NOT save that later-created copy.
#
if is_host
then
    DEVDB="/var/opt/hpvm/common/hpvm_mgmtdb"
    copy_file_step "Copying host's device database" \
                    "$DEVDB" \
                    config/${WHERE}.hpvm_mgmtdb
fi

# Run ioscan
tag_start "Collecting I/O configuration info"
new_cmd_log config/${WHERE}.ioscan
if is_hpux
then
    cmd_to_log ioscan -fkn
    # We really want additional data on 11.31 systems.
    if is_hpux_11iv3
    then
        cmd_to_log ioscan -fknN
        cmd_to_log /usr/sam/lbin/disklist -p -c /var/adm -r -t fs -y
    fi
    cmd_to_log /usr/sam/lbin/disklist -p -c /var/adm -r -t fs
    cmd_to_log /usr/sam/lbin/lvlist -p
else
    cmd_to_log systool -r system -v
    cmd_to_log systool -b pci -v
    cmd_to_log systool -c scsi_device -v
    cmd_to_log systool -c scsi_host -v
    cmd_to_log lspci -vvx
    cmd_to_log lshal
fi
tag_finish $ST_GOOD

# Run df, mount, swapinfo
tag_start "Collecting filesystem info"
new_cmd_log config/${WHERE}.fsinfo
hpux_only  cmd_to_log bdf -il
linux_only cmd_to_log df -il
linux_only cmd_to_log df -lahT
cmd_to_log mount -v
hpux_only  cmd_to_log swapinfo
linux_only cmd_to_log cat /proc/swaps
tag_finish $ST_GOOD

# run uname, model, uptime, osinfo, lsdev, machinfo, kconfig, kctune, kcusage
tag_start "Collecting system info"
new_cmd_log config/${WHERE}.sysinfo
cmd_to_log uname -a
hpux_only  cmd_to_log model
cmd_to_log uptime
# Warning! This next depends on a running cimserver, which seems to be an issue
# with Linux guests.... REVISIT-maybe
cmd_to_log osinfo
hpux_only  cmd_to_log lsdev
hpux_only  cmd_to_log machinfo -v
hpux_only  cmd_to_log mpsched -s
hpux_only  cmd_to_log kconfig -S
hpux_only  cmd_to_log kctune -g
hpux_only  cmd_to_log kcusage
hpux_only  cmd_to_log kcmodule
# This next added because these tunables are 'hidden'.
hpux_11iv3_only  cmd_to_log kctune base_pagesize lockable_mem_pct
linux_only cmd_to_log cat /proc/devices
linux_only cmd_to_log cat /proc/cpuinfo
linux_only cmd_to_log lsmod
linux_only cmd_to_log sysctl -a
host_only  cmd_to_log hpvmctrld -d
tag_finish $ST_GOOD

# run lanscan, netstat
tag_start "Collecting lan info"
new_cmd_log config/${WHERE}.laninfo
hpux_only  cmd_to_log lanscan
hpux_only  cmd_to_log lanscan -v
hpux_only  cmd_to_log lanscan -q
cmd_to_log netstat -in
linux_only cmd_to_log ifconfig -a
hpux_only  cmd_to_log arp -An
linux_only cmd_to_log arp -an
cmd_to_log netstat -rn
cmd_to_log netstat -s
if is_host
then
    cmd_to_log hpvmnet
    cmd_to_log hpvmnet -V
    SWITCHES=$(hpvmnet -M 2>/dev/null | cut -d: -f1)
    for SWITCH in $SWITCHES
    do
        cmd_to_log hpvmnet -S $SWITCH -V
    done
fi
tag_finish $ST_GOOD

# run lanshow if it exists
tag_start "Running lanshow"
lanshow -h > lanshow.exist 2>&1
NOTFOUND=$(grep "not found" lanshow.exist)
LANSHOWTARGET=""
rm -f lanshow.exist > /dev/null 2>&1
if [[ -z "$NOTFOUND" ]]
then
    if [[ -n "$cFLAG" ]]
    then
        mkdir -p crash > /dev/null 2>&1
        LANSHOWTARGET="crash/lanshow.log"
        lanshow -f "$CRASHDIR" > ${LANSHOWTARGET} 2>&1
    else
        LANSHOWTARGET="config/${WHERE}.lanshow"
        lanshow -f > ${LANSHOWTARGET} 2>&1
    fi
    if [[ -f ${LANSHOWTARGET} ]]
    then
	tag_finish $ST_GOOD
    else
	tag_finish $ST_FAIL
    fi
else
    tag_finish $ST_SKIP
fi


# run swlist
tag_start "Collecting installed sw info"
new_cmd_log config/${WHERE}.swinfo
hpux_only  cmd_to_log swlist
hpux_only  cmd_to_log swlist -l fileset -a state
linux_only cmd_to_log rpm -q -a
linux_only cmd_to_log rpm -q -a -i
tag_finish $ST_GOOD

#############################
# host specific collections #
#############################
if is_host
then

# collect command (API/CLI) logs
tag_start "Collecting command logs"
cp -p /var/opt/hpvm/common/command.log logs/ > /dev/null 2>&1 
cp -p /var/opt/hpvm/common/command.log.old logs/ > /dev/null 2>&1 
tag_finish $ST_GOOD

# collect hpvm driver log
tag_start "Collecting messages from vmm"
cp -p /var/opt/hpvm/common/hpvm_mon_log logs/ > /dev/null 2>&1 
# put older logs into old_hpvm_log/ directory under logs/
mkdir logs/old_hpvm_log > /dev/null 2>&1
cp -p /var/opt/hpvm/common/hpvm_mon_log.* logs/old_hpvm_log > /dev/null 2>&1
tag_finish $ST_GOOD

# Collect disk information 

# Capture disk information for the guest just once.  We do it this way so that
# a timeout on a 'stuck' running guest will still get us what's in
# vmm_config.current, since that'd be what's most relevant, and still only
# suffer the timeout once.
set -A DISKLIST $(hpvmstatus -P ${GUEST} -M 2>/dev/null | cut -d: -f22 | tr , ' ')

# Now, scan first for lvm-type lv's.
TDISKS=$( for TDISK in "${DISKLIST[@]}" ; do
            [[ "$TDISK" = *';/dev/vx/'* ]] && continue
            [[ "$TDISK" = *';lv;'* ]] || continue
            echo "${TDISK##*;}"
        done
        )
DISKS=""
for DISK in $TDISKS
do
	TDIRNAME=$(dirname "$DISK")
	SEDCMD="s,$TDIRNAME/r,$TDIRNAME/,"
	TDISK=$(echo $DISK | sed $SEDCMD)
	DISKS="$DISKS $TDISK"
done


tag_start "Collecting lv info"
new_cmd_log config/${WHERE}.diskinfo
if [[ -z "$DISKS" ]]
then
    cmd_to_log : lvdisplay
    tag_finish $ST_SKIP
else
    cmd_to_log lvdisplay $DISKS
    tag_finish $ST_GOOD
fi

# collect information for all supported volumes
# Get details for each volume under LVM control
LVINFO=config/${WHERE}.diskinfo
LVMVGS=$(vgdisplay | awk '{ if ($1 == "VG" && $2 == "Name") { print $3 } }')

tag_start "Collecting vgdisplay info"
new_cmd_log config/${WHERE}.diskinfo
if [[ -z "$LVMVGS" ]]
then
    cmd_to_log : vgdisplay
    tag_finish $ST_SKIP
else
    for I in $LVMVGS
    do
	cmd_to_log vgdisplay -v $I
    done
    tag_finish $ST_GOOD
fi

# Get details for each volume under VxVM control
tag_start "Collecting vxprint info"
new_cmd_log config/${WHERE}.diskinfo
cmd_to_log vxprint
cmd_to_log vxprint -Al
cmd_to_log vxdisk list
tag_finish $ST_GOOD


# diskinfo
# (for 'real' disks)
DISKS=$( for TDISK in "${DISKLIST[@]}" ; do
            [[ "$TDISK" = *';disk;'* ]] || continue
            echo "${TDISK##*;}"
        done
        )
tag_start "Collecting disk info"
new_cmd_log config/${WHERE}.diskinfo
if [[ -z "$DISKS" ]]
then
    cmd_to_log : diskinfo
    tag_finish $ST_SKIP
else
    for disk in $DISKS
    do
	cmd_to_log diskinfo -v $disk
    done
    tag_finish $ST_GOOD
fi

# diskinfo for scsi raw
# (do we still need this?!? REVISIT-maybe)
DISKS=$( for TDISK in "${DISKLIST[@]}" ; do
            [[ "$TDISK" = *';raw;'* ]] || continue
            echo "${TDISK##*;}" | sed 's,/rscsi/,/rdsk/,'
        done
        )
tag_start "Collecting passthru disk info"
new_cmd_log config/${WHERE}.diskinfo
cmd_to_log : pass through device diskinfo
if [[ -z "$DISKS" ]]
then
    tag_finish $ST_SKIP
else
    for disk in $DISKS
    do
        cmd_to_log diskinfo -v $disk
    done
    tag_finish $ST_GOOD
fi

# file pstore
DISKS=$( for TDISK in "${DISKLIST[@]}" ; do
            [[ "$TDISK" = *';file;'* ]] || continue
            echo "${TDISK##*;}"
        done
        )
tag_start "Collecting file backing store info"
cmd_to_log : file backing store
if [[ -z "$DISKS" ]]
then
    tag_finish $ST_SKIP
else
    for disk in $DISKS
    do
        cmd_to_log ls -lsi $disk
    done
    tag_finish $ST_GOOD
fi

# parse for "log" and get logfile entry
LOGFILE=$(sed -n -e 's/^[[:space:]]*log[[:space:]]*=[[:space:]]*//p' \
                $CONFIG_FILE)
case "$LOGFILE" in
""|stdout|stderr)
    # These aren't so useful ... stick with the default value.
    LOGFILE=$BROOT/log
    ;;
/*)
    # This one's already absolute, leave it alone
    ;;
*)
    # Guest-relative path, make absolute.
    LOGFILE=$BROOT/$LOGFILE
    ;;
esac
copy_file_step "Copying guest's log file" "$LOGFILE" logs/guest.$GUEST.Logfile

# get "tombstone" file if present
copy_file_step "Copying guest's tombstone file" \
                "$BROOT/tombstone" \
                logs/guest.$GUEST.tombstone

copy_file_step "Copying guest's console log file" \
                "$BROOT/console/conslog" \
                logs/guest.conslog.bin

copy_file_step "Copying hpvm configuration" \
                "/etc/rc.config.d/hpvmconf" \
                config/${WHERE}.hpvmconf

copy_file_step "Copying hpvm control script" \
                "/sbin/init.d/hpvm" \
                config/${WHERE}.initd.hpvm

tag_start "Copying guest's config file"
if [[ -f "$CONFIG_FILE" ]]
then
    cp -pf ${BROOT}/vmm_config.@(prev|current|next) config/ > /dev/null 2>&1 &&
    tag_finish $ST_GOOD ||
    tag_finish $ST_FAIL
else
    tag_finish $ST_SKIP
fi

tag_start "Getting status of the guest"
new_cmd_log config/hpvmstatus.out
cmd_to_log hpvmstatus -P $GUEST &&
    tag_finish $ST_GOOD ||
    tag_finish $ST_FAIL

echo "\n\n\n" >> $CMD_LOG

tag_start "Getting detailed status of the guest"
cmd_to_log hpvmstatus -VP $GUEST &&
    tag_finish $ST_GOOD ||
    tag_finish $ST_FAIL

echo "\n\n\n" >> $CMD_LOG

tag_start "Getting guest's entitlement"
cmd_to_log hpvmstatus -r -V -P $GUEST &&
    tag_finish $ST_GOOD ||
    tag_finish $ST_SKIP

tag_start "Copying guest's config file change log"
new_cmd_log config/hpvmstatus.changelog
cmd_to_log hpvmstatus -eVP $GUEST &&
    tag_finish $ST_GOOD ||
    tag_finish $ST_SKIP

copy_file_step "Copying guest VM crash image" $BROOT/vm.core images/
# The rest only matters if that was successful.
if [[ -f images/vm.core ]]
then
    if [[ -z "$cFLAG" ]]
    then
        DEFVMUNIX=/stand/current/vmunix
        if [[ -x /usr/sbin/kcpath ]]
        then
            VMUNIX=$(/usr/sbin/kcpath -x 2>/dev/null) || VMUNIX=
            # Ensure a defaulted value just in case.
            : "${VMUNIX:=$DEFVMUNIX}"
        else
            # hope it's in the standard place
            VMUNIX=$DEFVMUNIX
        fi
	copy_file_step "Copying host vmunix image" \
                        $VMUNIX \
                        images/vmunix
    fi
    copy_file_step "Copying host hpvmmkimage image" \
                    /opt/hpvm/bin/hpvmmkimage \
                    images/
fi

tag_start "Copying VMM image"
# We used to gzip it here, which is bad for crash analysis tools and for
# gzip'ing the archive itself.  Suppressing that for now.
if cp -pf /opt/hpvm/lbin/vmm images/ > /dev/null 2>&1
then
    # GZIP=-9 gzip -q images/vmm
    tag_finish $ST_GOOD
else
    tag_finish $ST_FAIL
fi

DEFSTANDPATH=/stand/current
if [[ -x /usr/sbin/kcpath ]]
then
    STANDPATH=$(/usr/sbin/kcpath -d 2>/dev/null) || STANDPATH=
    # Ensure a defaulted value just in case.
    : "${STANDPATH:=$DEFSTANDPATH}"
else
    STANDPATH=$DEFSTANDPATH
fi

copy_file_step "Copying hpvmdvr image" $STANDPATH/mod/hpvmdvr images/

copy_file_step "Copying hpvmntdvr image" $STANDPATH/mod/hpvmntdvr images/

# Should we copy the AVIO drivers, too?  REVISIT-maybe
# (Probably not useful when not already coming with a crash dump....)

NVRAM=$(sed -n -e 's,[[:space:]]*nvram[[:space:]]*=[[:space:]]*,,p' \
        $CONFIG_FILE)
copy_file_step "Copying NVRAM image" "$NVRAM" images/nvram

tag_start "Collecting IPMI logs"
typeset -i IPMI=0
cp -pf $BROOT/FPL logs/ > /dev/null 2>&1 && IPMI=IPMI+1
cp -pf $BROOT/SEL logs/ > /dev/null 2>&1 && IPMI=IPMI+1
if [[ "$IPMI" -eq 2  ]]
then
    tag_finish $ST_GOOD
elif [[ "$IPMI" -eq 1 ]]
then
    tag_finish PART		# REVISIT: should be $ST_FAIL (?)
else
    tag_finish $ST_SKIP
fi

tag_start "Copying guest debug memory file"
if [[ -d ${BROOT}/debug ]]
then
    typeset -i MEMDUMPNUM=0
    if [[ -f ${BROOT}/debug/memdump.bounds ]]
    then
	MEMDUMPNUM=$(<${BROOT}/debug/memdump.bounds)-1
    fi
    if [[ -f ${BROOT}/debug/memdump.${MEMDUMPNUM} ]]
    then
        mkdir images/debug > /dev/null 2>&1
        copy_file_finish ${BROOT}/debug/memdump.${MEMDUMPNUM} images/debug/
    else
	tag_finish $ST_SKIP
    fi
else
    tag_finish $ST_SKIP
fi

tag_start "Garbage collect guest debug memory files"
if [[ $GARBAGE -eq 1 ]]
then
    rm ${BROOT}/debug/memdump.* > /dev/null 2>&1 &&
	tag_finish $ST_GOOD ||
	tag_finish $ST_SKIP
else
    tag_finish $ST_SKIP
fi

########################################
# THE END OF host specific collections #
########################################
fi

# collect crash dump
if [[ -n "$cFLAG" ]]
then
    # We have $CRASHDIR as well as $CRASHNUM by now... which might be the same.
    # For Linux diskdump users, we can't be sure at all what CRASHNUM looks
    # like, so figure it out.
    # First, trim trailing delimiters.
    CRASHDIR="${CRASHDIR%%+(/)}"
    # Isolate the name in the directory.
    CRASHNAME="${CRASHDIR##*/}"
    # And, for HP-UX, re-isolate the number.
    CRASHNUM="${CRASHNAME#crash.}"

    tag_start "Collecting crash dump ${CRASHNUM}"

    mkdir -p crash > /dev/null 2>&1

    if [[ ! -d "$CRASHDIR" ]]
    then
	tag_finish $ST_FAIL
    else
        CRASHLOCAL=crash/$CRASHNAME
        mkdir "$CRASHLOCAL" > /dev/null 2>&1
        cp -pf "$CRASHDIR"/* "$CRASHLOCAL" \
            > /dev/null 2>&1
        if [[ -d "$CRASHLOCAL" ]]
        then
	    tag_finish $ST_GOOD
        else
	    tag_finish $ST_FAIL
        fi
    fi
else
    tag_start "Collecting crash dump"
    tag_finish $ST_SKIP
fi

# run "crashinfo" if possible
tag_start "Running crashinfo"
crashinfo -h > crashinfo.exist 2>&1
NOTFOUND=$(grep "not found" crashinfo.exist)
CRASHINFOTARGET=""
rm -f crashinfo.exist > /dev/null 2>&1
if [[ -z "$NOTFOUND" ]]
then
    if [[ -n "$cFLAG" ]]
    then
        mkdir -p crash > /dev/null 2>&1
        CRASHINFOTARGET="crash/crashinfo.log"
        crashinfo -v "$CRASHDIR" > ${CRASHINFOTARGET} 2>&1
    else
        CRASHINFOTARGET="config/${WHERE}.crashinfo"
        crashinfo -v > ${CRASHINFOTARGET} 2>&1
    fi

    if [[ -f ${CRASHINFOTARGET} ]]
    then
	tag_finish $ST_GOOD
    else
	tag_finish $ST_FAIL
    fi
else
    tag_finish $ST_SKIP
fi

# get tombstone for hardware problems
tag_start "Collecting tombstone"
TOMB=0
NA=0
if [[ -n "$cFLAG" ]]
then
    # The ts99 and ts98 stuff may be PA-specific.  Check what we've found on
    # IPF systems first.
    if [[ -f /var/tombstones/cpumap ]]
    then
        TOMB=1
        # Allow 3 all-cpu failures, plus the cpumap file.
        CPU_COUNT=$(getconf NUM_CPUS 2>/dev/null)
        if [[ -z "$CPU_COUNT" ]]
        then
            NA=1
        else
            typeset -i10 TOMB_LIMIT="1+3*CPU_COUNT"
            for file in $(ls -1r /var/tombstones/ | head -$TOMB_LIMIT)
            do
                cp -pf /var/tombstones/$file logs/ > /dev/null 2>&1 || NA=1
            done
        fi
    elif [[ -f /var/tombstones/ts99 ]]
    then
        cp -pf /var/tombstones/ts99 logs/ > /dev/null 2>&1
        TOMB=1
    else
        NA=1
    fi

    if [[ -f /var/tombstones/ts98 ]]
    then
        cp -pf /var/tombstones/ts98 logs/  > /dev/null 2>&1
        TOMB=1
    fi
fi

if [[ "$NA" -eq 1 ]]
then
    tag_finish $ST_FAIL
elif [[ "$TOMB" -eq 1 ]]
then
    tag_finish $ST_GOOD
else
    tag_finish $ST_SKIP
fi

tag_start "Collecting system message buffer"
dmesg > logs/${WHERE}.dmesg.log 2> /dev/null &&
tag_finish $ST_GOOD ||
tag_finish $ST_FAIL


tag_start "Collecting system syslogs"
if is_hpux
then
    cp -pf /var/adm/syslog/syslog.log logs/ > /dev/null 2>&1 
    cp -pf /var/adm/syslog/OLDsyslog.log logs/ > /dev/null 2>&1 
    cp -pf /etc/rc.log logs/  > /dev/null 2>&1
    cp -pf /etc/rc.log.old logs/  > /dev/null 2>&1
    cp -pf /var/adm/nettl.LOG* logs/ > /dev/null 2>&1
    cp -pf /var/adm/sw/swagent.log logs/ > /dev/null 2>&1
    tag_finish $ST_GOOD
else
    mkdir -p logs/varlog >/dev/null 2>&1 &&
    (cd /var/log ; tar cf - !(dump|dumps|crash|crashes)) |
        (cd logs/varlog ; tar xpf -) &&
    tag_finish $ST_GOOD ||
    tag_finish $ST_FAIL
fi


tag_start "Collecting measureware logs"
if [[ -d /var/opt/perf/datafiles ]]
then
    tar -cf logs/measureware.tar -C /var/opt/perf datafiles > /dev/null 2>&1 &&
    tag_finish $ST_GOOD ||
    tag_finish $ST_FAIL
else
    tag_finish $ST_SKIP
fi

if is_linux
then
    tag_start "Running sysreport -norpm"
    SYSREP=/usr/sbin/sysreport
    if [[ -f $SYSREP && -x $SYSREP ]]
    then
	new_cmd_log logs/sysreport.log
        # Note -- I'd love to have told sysreport that the right name for the
        # logfile is 'sysreport', but it's brain-dead about that.  It's already
        # using that directory name, and gets fatal 'mv' errors if you tell it
        # that for the name to use.  Thus this dodge of calling it
        # 'hpvmcollect' and fixing it when we move it into place.
	echo "x\nhpvmcollect" | $SYSREP -norpm > $CMD_LOG 2>&1
	SYSREPFILE=$(tail -5 $CMD_LOG | grep /tmp/ |
		     sed -e 's,^.* /,/,' -e 's, .*$,,')
	if [[ -f "$SYSREPFILE" ]]
	then
	    LOGFILE=$(echo $SYSREPFILE |
			sed -e 's,.*/,,' -e 's,hpvmcollect,sysreport,')
	    mv $SYSREPFILE $LOGFILE
	    tag_finish $ST_GOOD
	else
	    tag_finish $ST_FAIL
	fi
    else
	tag_finish $ST_SKIP
    fi
fi

TAG_TEE=

echo "\n  Finished.\n" >> $RLOG

################################
# Now finished with collecting #
################################
echo ""
echo "  Finished with the collection"
echo ""

tag_start "Tar archiving and compressing"
cd ..
if [[ "$ZIP" -eq 1 ]]
then

    # Tar everything except images directory first
    tar -cf $GDIR.tar $GDIR/!(images) > /dev/null 2>&1

    # Force images subdirectory to be tar'd at end
    tar -rf $GDIR.tar $GDIR/images > /dev/null 2>&1

    # do not gzip if -c option, unless copying to remote host
    if [[ -z "$cFLAG" || -n "$TARGETHOST" ]]
    then
       if [[ "$FORCE" -eq 1 ]]
       then    
            gzip -9qf $GDIR.tar >/dev/null 2>&1 && rm -rf $GDIR >/dev/null 2>&1
       elif [[ -f $GDIR.tar.gz ]]
       then
            mv $GDIR.tar.gz $GDIR.$DATE.tar.gz > /dev/null 2>&1
            gzip -9q $GDIR.tar >/dev/null 2>&1 && rm -rf $GDIR >/dev/null 2>&1
       else
            gzip -9q $GDIR.tar >/dev/null 2>&1 && rm -rf $GDIR >/dev/null 2>&1
       fi
    fi
    
    if [[ -f $GDIR.tar.gz ]]
    then
	tag_finish TGZ
        rm -rf $GDIR >/dev/null 2>&1
    elif [[ -f $GDIR.tar ]]
    then
	tag_finish TAR
        rm -rf $GDIR >/dev/null 2>&1
    else
	tag_finish FAIL
    fi
else
    tag_finish $ST_SKIP
fi

if [[ -z "$TARGETHOST" ]]
then
    TAGOUT=$ST_SKIP
else
    # we have to be very generous on invalid hostname
    # since we are gathering output from /dev/hpvmdvr, first
    # batch will be gone when hpvmcollect is run again.
    # simply indicate "HOST?" message if invalid hostname
    # (Actually, since we collect from the hpvm_mon_log file now, the above is
    # no longer true.  REVISIT-maybe.  This would also allow us to accept
    # user@host for those who'd like that.)
    if host "${TARGETHOST}" > /dev/null 2>&1
    then
        EXIST=$(/usr/bin/host $TARGETHOST | grep SERVFAIL)
        if [[ -z "$EXIST" ]]
        then
            echo ""
            TGTFILE="$HOSTNAME-guest.$GDIR.tar.gz"
            if [[ "$bFLAG" -eq "1" ]]
            then
                TARGETDIR="$TARGETROOT/$BUGRPT"
            else
                TARGETDIR="$TARGETROOT"
            fi
            echo " Copying to $TARGETHOST:$TARGETDIR/$TGTFILE"
            # Don't suppress errors here, since the user might want the clues
            # for fixing a remote access problem.
            if $RCP "$GDIR.tar.gz" "$TARGETHOST:$TARGETDIR/$TGTFILE"
            then
		TAGOUT=$ST_GOOD
            else
		TAGOUT=$ST_FAIL
            fi
        else
	    TAGOUT="HOST?"
        fi
    else
	TAGOUT="HOST?"
    fi
fi

tag_start "Remote copying the archive"
tag_finish "$TAGOUT"

echo "" 
if [[ -f $GDIR.tar.gz ]]
then
    echo "  The collection is\n  \"$WDIR/$GDIR.tar.gz\""
elif [[ -f $GDIR.tar ]]
then
    echo "  The collection is\n  \"$WDIR/$GDIR.tar\""
else
    echo "  The collection is in\n  \"$WDIR/$GDIR\""
fi
echo ""
