Newer
Older
tsshbatch / tsshbatch.py
#!/usr/bin/env python
# tsshbatch.py - Non-Interactive ssh Connection
# Copyright (c) 2011-2014 TundraWare Inc.
# Permission Hereby Granted For Unrestricted Personal Or Commercial Use
# See "tsshbatch-license.txt" For Licensing Details
#
# For Updates See:  http://www.tundraware.com/Software/tsshbatch

# A tip of the hat for some of the ideas in the program goes to:
#
#     http://jessenoller.com/2009/02/05/ssh-programming-with-paramiko-completely-different/

#####
# Program Housekeeping
#####

PROGNAME     = "tsshbatch.py"
BASENAME     = PROGNAME.split(".py")[0]
PROGENV      = BASENAME.upper()
CMDINCL      = PROGENV + "CMDS"
HOSTINCL     = PROGENV + "HOSTS"

CVSID        = "$Id: tsshbatch.py,v 1.208 2014/12/09 16:48:21 tundra Exp $"
VERSION      = CVSID.split()[2]
CPRT         = "(c)"
PROGDATE     = "2011-2014"
OWNER        = "TundraWare Inc."
RIGHTS       = "All Rights Reserved."
COPYRIGHT    = "Copyright %s %s, %s  %s" % (CPRT, PROGDATE, OWNER, RIGHTS)

PROGVER      = PROGNAME + " " + VERSION + (" - %s" % COPYRIGHT)
HOMEPAGE     = "http://www.tundraware.com/Software/%s\n" % BASENAME


#####
# Suppress Deprecation Warnings 
# Required in some older environments where paramiko version
# is behind the python libs version.
#####

import warnings
warnings.filterwarnings("ignore", "", DeprecationWarning)


#####
# Imports
#####

import commands
import getopt
import getpass
import os
import paramiko
import shlex
import socket
import sys
import time


#####
# Constants And Literals
#####


ABORTING    = 'Aborting ...'
COMMENT     = '#'
COMMANDS    = 'Commands'
CONSUCCESS  = 'SUCCESS: Connection Established'
EXECUTE     = '!'
FILEGET     = '-G'
FILEPUT     = '-P'
GETFILES    = 'Files To GET'
HOSTSEP     = '-'
HOSTNOISE   = '[%s]'
HOSTLIST    = 'Hosts'
INDENTWIDTH = 8
OPTIONSLIST = 'EKG:H:NP:ST:aef:hkl:n:p:qstvxy'
PADWIDTH    = 12
PATHDELIM   = ':'
PATHSEP     = os.sep
PUTFILES    = 'Files To PUT'
QUOTED      = '"'
QUOTES      = "'"
SEPARATOR   = ' --->  '
STDIN       = '-'
SUDO        = 'sudo'
SUDOPROMPT  = 'READINGSUDOPW'
SUDOARGS    = '-S -p %s' % SUDOPROMPT
SUDOPWHINT  = '(Default: login password): '
SYMTABLE    = 'Symbol Table'
TESTRUN     = 'Test Run For'
TRAILER     = ': '
USERVAR     = 'USER'

USAGE       = \
    PROGVER  + "\n"                                                                        +\
    HOMEPAGE + "\n\n"                                                                      +\
    "Usage:  tsshbatch.py [-EKNSTaehkqstvy -G 'file dest' -P 'file dest' -f cmdfile -l logfile -n name -p pw ] -H 'host ..' | hostlistfile [command arg ... ]\n" +\
    "          where,\n"                                                                   +\
    "\n"                                                                                   +\
    "            -E              Write error output to stdout instead of stderr\n"         +\
    "            -K              Force password prompting - Overrides previous -k\n"       +\
    "            -G 'file dest'  GET file on host and write local dest directory\n"        +\
    "            -H '...'        List of targeted hosts passed as a single argument\n"     +\
    "            -N              Force prompting for username\n"                           +\
    "            -P 'file dest'  PUT local file to host dest directory\n"                  +\
    "            -S              Force prompting for sudo password\n"                      +\
    "            -T seconds      Timeout for ssh connection attempts (Default: 15 sec)\n"  +\
    "            -a              Don't abort program after failed file transfers\n"        +\
    "            -e              Don't report remote host stderr output\n"                 +\
    "            -f cmdfile      Read commands from file\n"                                +\
    "            -h              Display help\n"                                           +\
    "            -k              Use key exchange-based authentication\n"                  +\
    "            -l logfile      Log errors to logfile (Default: /dev/null)\n"             +\
    "            -n name         Specify login name\n"                                     +\
    "            -p pw           Specify login password\n"                                 +\
    "            -q              Quiet mode - produce less 'noisy' output\n"               +\
    "            -s              Silence all program noise - only return command output\n" +\
    "            -t              Run in test mode, don't actually execute commands\n"      +\
    "            -v              Display extended program version information\n"           +\
    "            -x              Turn off test mode (if on) and execute requests\n"        +\
    "            -y              Turn on 'noisy' reporting for additional detail\n"


#####
# Directives & Related Support
#####

ASSIGN      = '='
DEFINE      = '.define'
INCLUDE     = '.include'


#####
# Builtin Definitions
#####

DATE        = '__DATE__'
DATETIME    = '__DATETIME__'
HOSTNAME    = '__HOSTNAME__'
HOSTNUM     = '__HOSTNUM__'
HOSTSHORT   = '__HOSTSHORT__'
LOGINNAME   = '__LOGINNAME__'
TIME        = '__TIME__'


# This is needed to differentiate between user-defined and builtin
# variables later in order to support the user being able to redefine
# them.

BuiltIns    = (DATE, DATETIME, HOSTNAME, HOSTNUM, HOSTSHORT, LOGINNAME, TIME)


#####
# Global Symbol Table
#####

SymbolTable = {}


#####
# Error Messages
#####

eBADARG       =  "Invalid command line: %s!"
eBADEXEC      =  "Execution variable failed: %s"
eBADFILE      =  "Cannot open '%s'!"
eBADSUDO      =  "sudo Failed (Check Password Or Command!) sudo Error Report:  %s"
eBADTXRQ      =  "Bad Transfer Request: %s  Must Have Exactly 1 Source And 1 Destination!"
eBADDEFINE    =  "Bad Symbol Definition: %s"
eBADTIMEOUT   =  "Timeout Value Must Be an Integer!"
eCMDFAILURE   =  "Failed To Run Command(s): %s"
eFXERROR      =  "File Transfer Error: %s"
eINCLUDECYCLE =  "Circular Include At: %s"
eNOCONNECT    =  "Cannot Connect: %s"
eNOHOSTS      =  "No Hosts Specified!"
eNOLOGIN      =  "Cannot Login! (Login/Password Bad?)"


#####
# Informational Messages
#####

iTXFILE = "Writing %s To %s ..."


#####
# Prompts
#####

pPASS = "Password: "
pSUDO = "%s Password: " % SUDO
pUSER = "Username (%s): "


#####
# Options That Can Be Overriden By User
####

ABORTONFXERROR = True         # Abort after a file transfer error
GETSUDOPW      = False        # Prompt for sudo password
Hosts          = []           # List of hosts to target
KEYEXCHANGE    = False        # Do key exchange-based auth?
LOGFILE        = "/dev/null"  # Where paramiko logging output goes
PROMPTUSERNAME = False        # Don't use $USER, prompt for username
PWORD          = ""           # Password
REDIRSTDERR    = False        # Redirect stderr to stdout
REPORTERR      = True         # Report stderr output from remote host
TESTMODE       = True         # Run program in test mode, don't actually execute commands
TIMEOUT        = 15           # Connection attempt timeout (sec)
UNAME          = ""           # Login name


# Noise levels

NOISELEVEL     = 0            # Normal noisy
SILENT         = 1            # No program noise at all
QUIET          = 2            # Make output less noisy
NOISY          = 3            # Print output with extra detail


#####
# Global Data Structures
#####

Commands          = []
FileIncludeStack  = []
Get_Transfer_List = {}
Put_Transfer_List = {}


#####
# Functions
#####

# Gets rid of comments and strips leading/trailing whitespace

def ConditionLine(line):
    return line.split(COMMENT)[0].strip()

# End of 'ConditionLine()'


#####
# Check To See If A Key Exists In A String, Excluding Quoted Substrings
#####

def KeyInString(key, string):

    """ Look for 'key' in 'string', but exclude segments of the string
        that are contained within single- or double quotes.
    """

    quote_chars = ('"', "'")

    InLiteral = False
    search = ""
    index = 0
    while index < len(string):

        char = string[index]

        if InLiteral:
            if char == quote_char:
                InLiteral = False
                
        elif char in quote_chars:
            quote_char = char
            InLiteral = True

        else:
            search += char

        index += 1
        
    status = False
    if search.count(key) > 0:
        status = True

    return status

# End of KeyInString


#####
# Print Message(s) To stderr
#####

def PrintStderr(msg, EOL="\n"):

    # If we've been told to redirect to stdout, do so instead

    if REDIRSTDERR:
        PrintStdout(msg, EOL)

    else:

        sys.stderr.write(msg + EOL)
        sys.stderr.flush()

# End of 'PrintStderr()'


#####
# Print Message(s) To stdout
#####

def PrintStdout(msg, EOL="\n"):
    sys.stdout.write(msg + EOL)
    sys.stdout.flush()

# End of 'PrintStdout()'


#####
# Display An Error Message And Exit
#####

def ErrorExit(msg):

    if msg:
        PrintStderr(msg)

    os._exit(1)

# End Of 'ErrorExit()'


#####
# Transfer Files To A Host
#####

def HostFileTransfer(host, user, pw, filelist, GET=False):

    try:
        ssh = paramiko.SSHClient()

        # Connect and run the command, reporting results as we go
    
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        if KEYEXCHANGE:
            ssh.connect(host, timeout=TIMEOUT)
        else:
            ssh.connect(host, username=user, password=pw, allow_agent=False, look_for_keys=False, timeout=TIMEOUT)

        sftp = ssh.open_sftp()

        for src in filelist:

            # Process any .define substitions

            srcfile = VarSub(src)

            for destdir in filelist[src]:
                
                # Process any .define substitutions

                destdir = VarSub(destdir)

                # Make sure we have a trailing path separator

                destination = destdir
                if destination[-1] != PATHSEP:
                    destination += PATHSEP

                if GET:
                    destination += host + HOSTSEP + os.path.basename(srcfile)
                    PrintStdout(iTXFILE %  (host + ":" + srcfile, destination))
                    sftp.get(srcfile, destination)
                    os.chmod(destination, sftp.stat(srcfile).st_mode)

                else:
                    destination += os.path.basename(srcfile)
                    PrintStdout(iTXFILE %  (srcfile, host + ":" + destination))
                    sftp.put(srcfile, destination)
                    sftp.chmod(destination, os.stat(srcfile).st_mode)

        sftp.close()
        ssh.close()

    except:

        PrintReport([host, eFXERROR % str(sys.exc_info()[1])], HANDLER=PrintStderr)

        try:
          sftp.close()
          ssh.close()

        except:
            pass

        # Do we continue or not?
        if ABORTONFXERROR:
            ErrorExit("")

# End of 'HostFileTransfer()'

    
def HostCommands(host, user, pw, sudopw, commands):

    # Figure out if we want report formatting

    Format = True
    if NOISELEVEL == SILENT:
        Format = False

    ssh = paramiko.SSHClient()

    # Connect and run the command, reporting results as we go
    
    try: 
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

        if KEYEXCHANGE:
            ssh.connect(host, timeout=TIMEOUT)

        else:
            ssh.connect(host, username=user, password=pw, allow_agent=False, look_for_keys=False, timeout=TIMEOUT)

        if NOISELEVEL !=  QUIET:
            PrintReport([host, CONSUCCESS], FORMAT=Format)

        # Run all requested commands

        for command in commands:

            # Dereference variables

            command = VarSub(command)

            # It's possible to get blank lines from stdin.
            # Ignore them.

            if not command:
                continue

            # If this is a sudo run, force password to be read
            # from stdin thereby avoiding fiddling around with ptys.

            if KeyInString(SUDO + " ", command):
                command = command.replace(SUDO, "%s %s" % (SUDO, SUDOARGS), 1)
          
            stdin, stdout, stderr = ssh.exec_command(command)

            # If doing a sudo command, send the password

            if KeyInString(SUDO + " ", command):
                stdin.write("%s\n" % sudopw)
                stdin.flush()
    
                # If all we see on stderr at this point is our original
                # prompt, then then the sudo promotion worked.  A bad
                # password or bad command will generate additional noise
                # from sudo telling us to try again or that there was a
                # command error.

                sudonoise = " ".join(stderr.readline().split(SUDOPROMPT)).strip()

                if sudonoise:                 # sudo had problems
                    PrintReport([host + " [%s]" % command, eCMDFAILURE % (eBADSUDO % sudonoise)] + ["\n"], HANDLER=PrintStderr)
                    ssh.close()
                    raise SystemExit
                    
            cmdreport = " [%s]" % command
            if NOISELEVEL == QUIET:
                cmdreport = ""

            PrintReport([host + " (stdout)" + cmdreport, "\n"] + stdout.readlines() + ["\n"], FORMAT=Format)

            if REPORTERR:
                PrintReport([host + " (stderr)" + cmdreport, "\n"] + stderr.readlines() + ["\n"], HANDLER=PrintStderr, FORMAT=Format)

    # Handle aborts
                
    except SystemExit:
        ErrorExit(ABORTING)

    # Catch authentication problems explicitly
        
    except paramiko.AuthenticationException:
        PrintReport([host, eCMDFAILURE % eNOLOGIN], HANDLER=PrintStderr)
    
    # Everything else is some kind of connection problem

    except:
        PrintReport([host, eCMDFAILURE % (eNOCONNECT % str(sys.exc_info()[1]))], HANDLER=PrintStderr)

    ssh.close()

# End of 'HostCommands()'


#####
# Print Report
#####

# Expects input as [host, success/failure message, result1, result2, ...]
# Uses print handler to stdout by default but can be overriden at call
# time to invoke any arbitrary handler function.

def PrintReport(results, HANDLER=PrintStdout, FORMAT=True):


    if FORMAT:

        hostname = results[0]
        HANDLER(SEPARATOR + hostname +
                TRAILER +
                (PADWIDTH - len(results[0])) * " " +
                results[1])

        # Prepend the host name if we've asked for noisy reporting

        hostnoise =""
        if NOISELEVEL == NOISY:
            hostnoise = HOSTNOISE % hostname

        for r in results[2:]:                             # Command Results
            HANDLER(hostnoise + INDENTWIDTH * " " + r.strip())

    # In (silent) unformatted mode we just return results with no
    # headers or formatting.  We suppress the last line in the list.
    # It is an empty line introduced for formatting by the caller, but
    # isn't really part of the returned output.
    else:

        for r in results[2:-1]:
            HANDLER(r, EOL="")

# End of 'PrintReport()'


#####
# Process A File Transfer Request
#####

def ProcessTXRQ(request, storage):

    src_dest = request.split()
    if len(src_dest) != 2:
        ErrorExit(eBADTXRQ % src_dest)

    else:

        if src_dest[0] not in storage:
            storage[src_dest[0]] = [src_dest[1],]

        else:
            storage[src_dest[0]].append(src_dest[1])

# End of 'ProcessTXRQ'


#####
# Read File Handling Comments And Directives
#####

def ReadFile(fname, envvar, listcontainer, containingfile=""):

    # Check to see if we can find the file, searching the
    # the relevant include environment variable path, if any

    filename = SearchPath(fname, envvar)
    if not filename:
        ErrorExit(eBADFILE % fname)

    # Make sure we don't have a cyclic include reference

    if filename in FileIncludeStack:
        ErrorExit(eINCLUDECYCLE % containingfile + SEPARATOR + filename)

    else:
        FileIncludeStack.append(filename)  # Push it on to the stack history

    # Line parsing starts here
    try:

        f = open(filename)
        for line in f.readlines():
            
            # Cleanup comments and whitespace

            line = ConditionLine(line)
            
            # Process file transfer requests

            if line.startswith(FILEGET):
                val = line.replace(QUOTED, '').replace(QUOTES, '').split(FILEGET)[1].strip()
                ProcessTXRQ(val, Get_Transfer_List)

            elif line.startswith(FILEPUT):
                val = line.replace(QUOTED, '').replace(QUOTES, '').split(FILEPUT)[1].strip()
                ProcessTXRQ(val, Put_Transfer_List)

            # Process variable definitions

            elif line.startswith(DEFINE):
                
                line = line.split(DEFINE)[1]
                if line.count(ASSIGN) == 0:
                    ErrorExit(eBADDEFINE % line)

                else:

                    name = line.split(ASSIGN)[0].strip()
                    val  = "=".join(line.split(ASSIGN)[1:]).strip()

                    # Process references to execution variables

                    if val.startswith(EXECUTE):

                        try:

                            origval = val
                            status, val = commands.getstatusoutput(val[1:].strip())

                            # Blow out if the command failed

                            if status:
                                raise

                        except:

                            PrintReport([PROGNAME, eBADEXEC % origval], HANDLER=PrintStderr)
                            ErrorExit("")

                    if name:
                        SymbolTable[name] = val

                    else:
                        ErrorExit(eBADDEFINE % line)

            # Process file includes
            elif line:
                if line.startswith(INCLUDE):
                    fname = ConditionLine(line.split(INCLUDE)[1])
                    ReadFile(fname, envvar, listcontainer, containingfile=filename)

                # It's a normal line - do variable substitution and save
                else:
                    listcontainer.append(line)
        f.close()

        FileIncludeStack.pop()   # Remove this invocation from the stack
        return listcontainer

    except:
        ErrorExit(eBADFILE % filename)

# End of 'ReadFile()'


#####
# Search A Path For A File, Returning First Match
#####

def SearchPath(filename, pathlist, delimiter=PATHDELIM):

    # What we'll return if we find nothing
    retval = ""

    # Handle fully qualified filenames
    # But ignore this, if its a directory with a matching name
    
    if os.path.exists(filename) and os.path.isfile(filename):
        retval =  os.path.realpath(filename)

    # Find first instance along specified path if one has been specified
    elif pathlist:

        paths = pathlist.split(delimiter)
        for path in paths:

            if path and path[-1] != PATHSEP:
                path += PATHSEP

                path += filename

                if os.path.exists(path):
                    retval = os.path.realpath(path)
                    break
    return retval

# End of 'SearchPath()'


#####
# Do Variable Substitution In A String
#####

def VarSub(line):

    for symbol in SymbolTable:
        line = line.replace(symbol, SymbolTable[symbol])
        
    return line

# End of 'VarSub()'


# ---------------------- Program Entry Point ---------------------- #

#####
# Process Any Options User Set In The Environment Or On Command Line
#####

# Handle any options set in the environment

OPTIONS = sys.argv[1:]
envopt = os.getenv(PROGENV)
if envopt:
    OPTIONS = shlex.split(envopt) + OPTIONS

# Combine them with those given on the command line
# This allows the command line to override defaults
# set in the environment

try:
    opts, args = getopt.getopt(OPTIONS, OPTIONSLIST)

except getopt.GetoptError, (errmsg, badarg):
    ErrorExit(eBADARG % errmsg)

for opt, val in opts:


    if opt == "-E":
        REDIRSTDERR = True

    if opt == "-K":
        KEYEXCHANGE = False

    if opt == FILEGET:
        ProcessTXRQ(val, Get_Transfer_List)

    if opt == "-H":
        Hosts = val.split()
        
    if opt == "-N":
        PROMPTUSERNAME = True
        KEYEXCHANGE = False
        
    if opt == FILEPUT:
        ProcessTXRQ(val, Put_Transfer_List)

    if opt == "-S":
        GETSUDOPW = True

    if opt == "-T":
        try:
            TIMEOUT = int(val)
        except:
            ErrorExit(eBADTIMEOUT)

    if opt == "-a":
        ABORTONFXERROR = False
        
    if opt == "-e":
        REPORTERR = False
        
    if opt == "-f": 
        Commands = ReadFile(val, os.getenv(CMDINCL), Commands)
    
    if opt == "-h":
        PrintStdout(USAGE)
        sys.exit()
        
    if opt == "-k":
        KEYEXCHANGE = True

    if opt == "-l":
        LOGFILE = val

    if opt == "-n":
        UNAME = val

    if opt == "-p":
        PWORD = val

    if opt == "-q":
        NOISELEVEL = QUIET

    if opt == "-s":
        NOISELEVEL = SILENT

    if opt == "-t":
        TESTMODE = True

    if opt == "-v":
        PrintStdout(CVSID)
        sys.exit()

    if opt == "-x":
        TESTMODE = False

    if opt == "-y":
        NOISELEVEL = NOISY


#####
# Intitialize paramiko Logging
#####

paramiko.util.log_to_file(LOGFILE)


#####
# Host & Command Line Command Definition Processing
#####

# Get the list of hosts if not specified on command line.
# The assumption is that the first argument is the file
# containing the list of targeted hosts and the remaining
# arguments form the command.

if not Hosts:

    # Even if we are only doing file transfers and no command
    # is specified, we have to have at least one argument here
    # to tell us what hosts we're working on.

    if not args:
        ErrorExit(eNOHOSTS)

    Hosts = ReadFile(args[0], os.getenv(HOSTINCL), Hosts)
    command = " ".join(args[1:])
    
# If hosts were passed on the command line, all the arguments
# are understood to form the command.
    
else:
    command = " ".join(args[0:])

# Put it in a list data structure because this is what the
# HostCommands() function expects.  This is necessary to handle multi
# command input from from a file.

command = ConditionLine(command)

if command:

    # Do variable substitution here like any other command
    Commands.append(command)

#####
# Authentication Credential Processing
#####

# Precedence of authentication credential sources:
#
#     1) Key exchange
#     2) Forced prompting for name via -N
#     3) Command Line/$TSSHBATCH env variable sets name
#     4) Name picked up from $USER  (Default behavior)

if not KEYEXCHANGE:

    # Preset commandline and/or program option variable username takes precedence

    if not UNAME:
        UNAME = os.getenv(USERVAR)
        
    # By default, use the above as the login name and don't prompt for it
    # unless overriden on the command line with -N

    if PROMPTUSERNAME:

        current_user = UNAME
        UNAME = raw_input(pUSER %current_user)
        if not UNAME:                 # User just hit return - wants default
            UNAME = current_user

    # Preset commandline and/or program option variable password takes precedence

    if not PWORD:
        PWORD  = getpass.getpass(pPASS)

#####
# If Needed, Get sudo Password
####

# The need to prompt for a sudo password depends on a number of
# conditions:
#
# If a login password is present either via manual entry or -p, sudo
# will use that without further prompting.  (Default)
#
# The user is prompted for a sudo password under two conditions:
#
#  1) -k option was selected but no password was set with -p
#  2) -S option was selected 
#
# If the user IS prompted for a sudo password, any login password
# previously entered - either via -p or interactive entry - will be
# used as the default.  The user can hit enter to accept this or enter
# a different password.  This allows login and sudo passwords to be
# the same or different.

# Find out if we have any sudo commands

SUDOPRESENT = False
for command in Commands:
    if KeyInString(SUDO + " ", command):
            SUDOPRESENT = True

# Check condition 1) above.
# (Condition 2 handled during options processing).

if KEYEXCHANGE and not PWORD:
    GETSUDOPW = True

SUDOPW = PWORD
if SUDOPRESENT and GETSUDOPW:

    sudopwmsg   = pSUDO
    if PWORD:
        sudopwmsg = sudopwmsg[:-2] + " " + SUDOPWHINT

    SUDOPW = getpass.getpass(sudopwmsg)
    if PWORD and not SUDOPW:
        SUDOPW = PWORD

#####
# Do The Requested Work
#####

# If we're running testmode, just report the final list of
# hosts and commands that would be run

if TESTMODE:

    symtbl = []
    gets   = []
    puts   = []

    # Unroll and format dictionary structures

    symbols = SymbolTable.keys()
    symbols.sort()
    for symbol in symbols:
        symtbl.append(symbol + (PADWIDTH - len(symbol)) * " "+ SEPARATOR + SymbolTable[symbol])

    for xfers, unrolled in ((Get_Transfer_List, gets), (Put_Transfer_List, puts)):

        for source in xfers:
            for dest in xfers[source]:

                # Dereference any variables in the file transfer specification
                
                source = VarSub(source)
                dest   = VarSub(dest)

                unrolled.append(source + (PADWIDTH*3 - len(source)) * " "+ SEPARATOR + dest)

    # Dereference any variables in the list of commands

    index = 0
    while (index < len(Commands)):

        Commands[index] = VarSub(Commands[index])
        index += 1

    # Dereference any variables in the list of hosts

    index = 0
    while (index < len(Hosts)):

        Hosts[index] = VarSub(Hosts[index])
        index += 1

    for prompt, description, items in ((TESTRUN,  " ".join(OPTIONS), ["\n"]),
                                       (SYMTABLE, "",                 symtbl + ["\n"]),
                                       (HOSTLIST, "",                  Hosts + ["\n"]),
                                       (GETFILES, "",                   gets + ["\n"]),
                                       (PUTFILES, "",                   puts + ["\n"]),
                                       (COMMANDS, "",               Commands + ["\n"])
                                      ):

        PrintReport([prompt, description] + items)


# Otherwise, actually do the work by iterating over the list of hosts,
# executing any file transfers and commands.  Accomodate commenting
# out hosts in a list.

else : 

    # Check to see if user is trying to override any builtins

    protected = []
    for builtin in BuiltIns:
        if builtin in SymbolTable:
            protected.append(builtin)

    # Now iterate over requested hosts
    hostnum = 0
    for host in Hosts:

        # Update the host counter

        hostnum += 1

        # Add internally generated symbols to the symbol table.
        # That way, both user-defined and builtin symbols will
        # subsequently be substituted.

        # Find out the effective name of the user doing this

        uname = UNAME
        if not uname:
            uname = os.getenv(USERVAR)

        internals = [
                     (DATE, time.strftime("%Y%m%d")),
                     (DATETIME, time.strftime("%Y%m%d%H%M%S")),
                     (HOSTNAME, host),
                     (HOSTNUM, str(hostnum)),
                     (HOSTSHORT, host.split('.')[0]),
                     (LOGINNAME, uname),
                     (TIME, time.strftime("%H%M%S")),
                    ]

        # Install builtins in the symbol table but only if the
        # user isn't overriding them.

        for symbol, value in internals:
            if symbol not in protected:
                SymbolTable[symbol] = value

        # Apply any relevant variable dereferencing

        host = VarSub(host)

        if Get_Transfer_List:
            HostFileTransfer(host, UNAME, PWORD, Get_Transfer_List, GET=True)

        if Put_Transfer_List:
            HostFileTransfer(host, UNAME, PWORD, Put_Transfer_List, GET=False)

        if Commands:
            HostCommands(host, UNAME, PWORD, SUDOPW, Commands)