blades-ssh-scraper/main.py

# External deps
import paramiko # for ssh
import logging # for possible debugging
from time import sleep # for sleep()
import os # for CLI arguments

# Local deps
import listServers, relativeUsage

# Debugging stuffs
#logging.basicConfig()
#logging.getLogger("paramiko").setLevel(logging.DEBUG)

"""
ENV VARIABLES AVAILABLE:
 - hostname
 - sshkeylocation (optional)
 - sshuser (optional)
"""
# Creds
hostname = str(os.environ['hostname'])
sshuser   = str(os.getenv('sshuser', 'Administrator'))
sshkeylocation   = str(os.path.abspath( os.getenv('sshkeypath', '~/.ssh/id_rsa') ))

paramiko.transport.Transport._preferred_keys += ('ssh-dss',) # Allow the insecurities
ssh = paramiko.SSHClient() # Alias
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # Ignore host key REMOVE ME (though it doesn't kind of matter anyways)

ssh.connect(hostname, username=sshuser, key_filename=sshkeylocation) # Attempt to connect
channel = ssh.invoke_shell() # Get a new shell(, and keep it open, since we need to exec multiple commands depending on the last command being executed successfully.)

def ssh_runcmd(command): # Some abstraction to run ssh commands.
    # Input validation
    if not type(command) is str:
        raise TypeError("ssh_runcmd: command must be a string, '" + command + "' is not.")

    channel.send(command + '\n') # Execute command.

    cmdout = "" # Init
    lockedoutCounter = 0 # Init

    while channel.recv_ready() or not cmdout[-2:] == '> ': # Fetch new output, if there is any or look for '> ' (prompt ready). The latter is useful when connecting to blades.
        """
        SSH to a blade from the enc takes about 6-7s. If we are at 15s, something is probably wrong. connect servers fail due to:
        a) There are too many users on SSH (ungraceful sessions?)
        b) The iLO is unavailable (has been just reset)
        c) The blade has been just plugged in (not initialized)
        d) SSH access has been disabled
        e) other
        All but e) are known to drop you back in to the enc's shell, in what case we'll be in a problematic spot, since we're going to exit out of the enc ssh, not the blade's.
        #TODO: somehow magically detect if we've successfully connected to the blade. There's no standard way to determine.
        The probable way would be to look at the prompt. Blade's seem to all be '</>hpiLO-> '.
        I would fetch the prompt on connection to the enc, then compare the current prompt to determine if we're connected to a blade.
        """
        lockedoutCounter += 1
        if lockedoutCounter > 30:
            raise EOFError("Waited for a prompt for 15s. Something is probably wrong. Aborting.")
        else:
            logging.info("No prompt, waiting for more data.")
            sleep(0.5)
            cmdout += channel.recv(65536).decode("ascii") # Get output, capped to 64KiB, format ascii. | Must get, otherwise next command will get this command's output.
    #breakpoint()

    cmdout = cmdout.split('\n')[2:-3] # Split to a list, per newlines. Remove first 2, and last 3 lines.
    #cmdout = '\n'.join(cmdout) # DO NOT UNCOMMENT THIS, HERE FOR ONLY REFERENCE TO JOIN THE LINES BACK.
    return(cmdout) # Return list of output stuff.

### MAIN ###

logging.debug(ssh_runcmd('show date')) # Get rid of motd, init for next cmds. This is better than indefinitely reading buffer before any command as to counter this. 


## Get list of blades with some added info ##
serverName = ssh_runcmd('show server names') # Testing data: serverName = ['Bay Server Name                                       Serial Number   Status   Power   UID Partner\r', '--- ------------------------------------------------- --------------- -------- ------- --- -------\r', '  1 tty-lab-1                                                         OK       On      Off \r', '  2 tty-lab-2                                         CZ320263P9      OK       On      Off \r', '  3 tty-lab-3                                         CZJ14410KP      Failed   On      Off \r', '  4 kspve1                                            CZJ14410KK      OK       On      Off \r', '  5 kspve2-2                                                          OK       On      Off \r', '  6 kspve3                                                            OK       On      Off \r', '  7 plaes-blade                                                       OK       On      Off \r', '  8 Ringly-01                                         CZ3402Y48C      OK       On      Off \r', '  9 toomas-lepik                                      CZ3217FNYE      OK       On      Off \r', ' 10 toomas-lepik2                                     CZ3217FFSS      OK       On      Off \r', ' 11 [Absent]                                          \r', ' 12 erki-naumanis                                                     OK       On      Off \r', ' 13 [Absent]                                          \r', ' 14 [Absent]                                          \r', ' 15 [Absent]                                          \r', ' 16 [Absent]                                          \r']
servers = listServers.listServers(serverName)

baysInUse = [x[0] for x in servers] # List of blades in use.
baysInUseCount = len(baysInUse) # How many bays in use.
logging.info("There are " + str(baysInUseCount+1) + "servers presenet.") # Further optimizations could be made by not connecting to servers, what are turned off.


## Get blade data ##
for n in range(baysInUseCount):
    bay = servers[n][0] # We want the bay, not how many times we have looped over.
    logging.info("Accessing server " + str(bay))
    logging.debug( ssh_runcmd('connect server ' + str(bay)) ) # Use the enc as a jump host.
    powerInfoTmp = ssh_runcmd('show system1/oemhp_power1') # Get the data
 
    presentPower = [i for i in powerInfoTmp if i.startswith('    oemhp_PresentPower=')][0][23:-7] # Get the line with PresentPower, then remove first 23, and last 7 chars to end up with the Watts DC the blade is directly using.
    servers[n].append(int(presentPower)) # And push it to our miniDB of the servers list.
    logging.info("UsageRawDC: " + presentPower)

    logging.debug(ssh_runcmd('exit')) # Exit the blade's iLO, return to the enc's iLO SSH interface.
    #breakpoint()

#print(servers)
#breakpoint()
# Fields: [BayNumber "Server Name", "Serial Number", "Status", "Power", "UID Partner", presentPowerDirectDC]


## Get enc's _AC_ usage. ##
encPowerUsageAC = int(str.strip( [i for i in ssh_runcmd('show power') if i.startswith('Present Power:')][0][14:-9] ))
logging.info("Server total usage AC: " + str(encPowerUsageAC))

## End sesion with enc. ##
channel.send('exit' + '\n') # Using lower level cmd since we're not going to get a prompt back here.
ssh.close()


## Calculating blade usage by percentage. ## 
# Keep in mind the querying of the data took a while, a minute or so.
serversAverageACAndServers = relativeUsage.relativeUsage(encPowerUsageAC, baysInUseCount, servers)
serversAverageAC = serversAverageACAndServers[0]
servers = serversAverageACAndServers[1]

# Fields: [BayNumber "Server Name", "Serial Number", "Status", "Power", "UID Partner", presentPowerDirectDC, relativeUsageAC]

print("Enclosure usage AC: " + str(encPowerUsageAC))
print("Average blade relative usage AC: " + str(serversAverageAC))
print('[BayNumber "Server Name", "Serial Number", "Status", "Power", "UID Partner", presentPowerDirectDC, relativeUsageAC]')
print(servers)