blades-ssh-scraper/scraperMain.py

# External deps
import paramiko # for ssh
import logging # for possible debugging
from time import sleep # for sleep()
from warnings import warn # you guessed it
import os # for CLI arguments

# Local deps
import listServers, relativeUsage

# Debugging stuffs
#logging.basicConfig()
#logging.getLogger("paramiko").setLevel(logging.DEBUG)

def ssh_runcmd(command): # Some abstraction to run ssh commands.
    # Input validation
    if not type(command) is str:
        raise TypeError("ssh_runcmd: command must be a string, '" + command + "' is not.")
    #TODO: valiate channel, though not really

    channel.send(command + '\n') # Execute command.

    cmdout = "" # Init
    lockedoutCounter = 0 # Init

    while channel.recv_ready() or not cmdout[-2:] == '> ': # Fetch new output, if there is any or look for '> ' (prompt ready). The latter is useful when connecting to blades.
        """
        SSH to a blade from the enc takes about 6-7s. If we are at 15s, something is probably wrong. connect servers fail due to:
        a) There are too many users on SSH (ungraceful sessions?)
        b) The iLO is unavailable (has been just reset)
        c) The blade has been just plugged in (not initialized)
        d) SSH access has been disabled
        e) other
        All but e) are known to drop you back in to the enc's shell, in what case we'll be in a problematic spot, since we're going to exit out of the enc ssh, not the blade's.
        #TODO: somehow magically detect if we've successfully connected to the blade. There's no standard way to determine.
        The probable way would be to look at the prompt. Blade's seem to all be '</>hpiLO-> '.
        I would fetch the prompt on connection to the enc, then compare the current prompt to determine if we're connected to a blade.
        """
        lockedoutCounter += 1
        if lockedoutCounter > 30:
            raise EOFError("Waited for a prompt for 15s. Something is probably wrong. Aborting.")
        else:
            logging.info("No prompt, waiting for more data.")
            sleep(0.5)
            cmdout += channel.recv(65536).decode("ascii") # Get output, capped to 64KiB, format ascii. | Must get, otherwise next command will get this command's output.
    #breakpoint()

    cmdout = cmdout.split('\n')[2:-3] # Split to a list, per newlines. Remove first 2, and last 3 lines.
    #cmdout = '\n'.join(cmdout) # DO NOT UNCOMMENT THIS, HERE FOR ONLY REFERENCE TO JOIN THE LINES BACK.
    return(cmdout) # Return list of output stuff.

def scraperMain(hostname, encname, sshkeypath):
    global channel # needed for ssh_runcmd
    # Input validation
    if not type(hostname) is str:
        raise TypeError("scraperMain: hostname must be a string, '" + str(hostname) + "' is not.")
    if not type(encname) is str:
        raise TypeError("scraperMain: enc must be a string, '" + str(encname) + "' is not.")
    if not type(sshkeypath) is str:
        raise TypeError("scraperMain: sshkeypath must be a string, '" + str(sshkeypath) + "' is not.")


    # ssh_init() # it takes more lines to put it in a function than to dump it here.
    sshuser = 'Administrator' # Admin is hardcoded in to the enc.

    paramiko.transport.Transport._preferred_keys += ('ssh-dss',) # Allow the insecurities
    ssh = paramiko.SSHClient() # Alias
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # Ignore host key REMOVE ME (though it doesn't kind of matter anyways)

    ssh.connect(hostname, username=sshuser, key_filename=sshkeypath) # Attempt to connect
    channel = ssh.invoke_shell() # Get a new shell(, and keep it open, since we need to exec multiple commands depending on the last command being executed successfully.)


    ## MAIN ##
    logging.debug(ssh_runcmd('show date')) # Get rid of motd, init for next cmds. This is better than indefinitely reading buffer before any command as to counter this. Reading buffer before each command isn't a good idea and doesn't work either, trust me.


    ## Get list of blades with some added info ##
    serverName = ssh_runcmd('show server names') # Testing data: serverName = ['Bay Server Name                                       Serial Number   Status   Power   UID Partner\r', '--- ------------------------------------------------- --------------- -------- ------- --- -------\r', '  1 foo-lab-1                                                         OK       On      Off \r', '  2 foo-lab-2                                         CZ302243P9      OK       On      Off \r', '  3 foo-lab-3                                         CZJI441OKP      Failed   On      Off \r', '  4 kspve1                                            CZJ18450FK      OK       On      Off \r', '  5 kspve2-2                                                          OK       On      Off \r', '  6 kspve3                                                            OK       On      Off \r', '  7 foo-blade                                                       OK       On      Off \r', '  8 Bar-01                                         CZ241274CC      OK       On      Off \r', '  9 baz-sar                                      CZ3217FNYE      OK       On      Off \r', ' 10 baz-sar2                                     CZ3217FFSS      OK       On      Off \r', ' 11 [Absent]                                          \r', ' 12 bee-bar                                                     OK       On      Off \r', ' 13 [Absent]                                          \r', ' 14 [Absent]                                          \r', ' 15 [Absent]                                          \r', ' 16 [Absent]                                          \r']
    servers = listServers.listServers(serverName)

    baysInUse = [x[0] for x in servers] # List of blades in use.
    baysInUseCount = len(baysInUse) # How many bays in use.
    logging.info("There are " + str(baysInUseCount+1) + "servers presenet.") # Further optimizations could be made by not connecting to servers, what are turned off.


    ## Get blade data ##
    for n in range(baysInUseCount):
        bay = servers[n][0] # We want the bay, not how many times we have looped over.
        logging.info("Accessing server " + str(bay))
        logging.debug( ssh_runcmd('connect server ' + str(bay)) ) # Use the enc as a jump host.
        powerInfoTmp = ssh_runcmd('show system1/oemhp_power1') # Get the data
    
        presentPower = [i for i in powerInfoTmp if i.startswith('    oemhp_PresentPower=')][0][23:-7] # Get the line with PresentPower, then remove first 23, and last 7 chars to end up with the Watts DC the blade is directly using.
        if presentPower == 'Not Ava': # When the server is powered off, it's not zero (LIKE LITERALLY EVERYWHERE ELSE). It is NoT aVaILAbLe. 'Not Ava' because the previous command takes last 7 chars (normally 'Watts') off of it. Ah my fault for not checking all possible input from hpe.
            presentPower = 0
        elif not presentPower.isdigit(): # Just in case some other anomalities come up in the future, like negative power draw or something…
            warn('presentPower for bay ' + bay + 'was corrected to 0 from it\'s abnormal state: ' + presentPower)
            presentPower = 0
        servers[n].append(int(presentPower)) # And push it to our miniDB of the servers list.
        logging.info("UsageRawDC: " + str(presentPower))

        logging.debug(ssh_runcmd('exit')) # Exit the blade's iLO, return to the enc's iLO SSH interface.
        #breakpoint()

    #print(servers)
    #breakpoint()
    # Fields: [BayNumber "Server Name", "Serial Number", "Status", "Power", "UID Partner", presentPowerDirectDC]


    ## Get enc's _AC_ usage. ##
    encPowerUsageAC = int(str.strip( [i for i in ssh_runcmd('show power') if i.startswith('Present Power:')][0][14:-9] ))
    logging.info("Server total usage AC: " + str(encPowerUsageAC))

    ## End sesion with enc. ##
    channel.send('exit' + '\n') # Using lower level cmd since we're not going to get a prompt back here.
    ssh.close()


    ## Calculating blade usage by percentage. ## 
    # Keep in mind the querying of the data took a while, a minute or so.
    serversAverageACAndServers = relativeUsage.relativeUsage(encPowerUsageAC, baysInUseCount, servers)
    #serversAverageAC = serversAverageACAndServers[0] # I guess you don't want any stonks by me, but prom instead…
    servers = serversAverageACAndServers[1]
    # Fields: [BayNumber "Server Name", "Serial Number", "Status", "Power", "UID Partner", presentPowerDirectDC, relativeUsageAC]

    for n in range(baysInUseCount):
            servers[n].append( encname ) # Add what enc was used.
    # Fields: [BayNumber "Server Name", "Serial Number", "Status", "Power", "UID Partner", presentPowerDirectDC, relativeUsageAC, "enc"]
    
    return(baysInUseCount, servers) # we could return more, but nobody wants our data :/

    #print("Enclosure usage AC: " + str(encPowerUsageAC))
    #print("Average blade relative usage AC: " + str(serversAverageAC))
    #print('[BayNumber "Server Name", "Serial Number", "Status", "Power", "UID Partner", presentPowerDirectDC, relativeUsageAC, "enc"]')
    #print(servers)