115 lines
6.8 KiB
Python
115 lines
6.8 KiB
Python
# External deps
|
|
import paramiko
|
|
import logging
|
|
from time import sleep
|
|
|
|
# Local deps
|
|
import listServers, relativeUsage
|
|
|
|
# Debugging stuffs
|
|
#logging.basicConfig()
|
|
#logging.getLogger("paramiko").setLevel(logging.DEBUG)
|
|
|
|
# Creds
|
|
hostname = '172.23.224.40'
|
|
sshuser = 'Administrator'
|
|
sshkeylocation = '/home/jc/.ssh/id_rsa'
|
|
|
|
paramiko.transport.Transport._preferred_keys += ('ssh-dss',) # Allow the insecurities
|
|
ssh = paramiko.SSHClient() # Alias
|
|
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) # Ignore host key REMOVE ME (though it doesn't kind of matter anyways)
|
|
|
|
ssh.connect(hostname, username=sshuser, key_filename=sshkeylocation) # Attempt to connect
|
|
channel = ssh.invoke_shell() # Get a new shell(, and keep it open, since we need to exec multiple commands depending on the last command being executed successfully.)
|
|
|
|
def ssh_runcmd(command): # ssh_runcmd <command>
|
|
# Input validation
|
|
if not type(command) is str:
|
|
raise TypeError("ssh_runcmd: command must be a string, '" + command + "' is not.")
|
|
|
|
channel.send(command + '\n') # Execute command.
|
|
|
|
while not channel.recv_ready(): # Wait for output to be recieved.
|
|
sleep(0.5) # The command could still be writing to output buffer after running for 0.5s. Assuming during the period we read the buffer, cmd can keep up with us, or at least till the 8th KiB end.
|
|
|
|
cmdout = "" # Init
|
|
lockedoutCounter = 0 # Init
|
|
|
|
while channel.recv_ready() or not cmdout[-2:] == '> ': # Fetch new output, if there is any or look for '> ' (prompt ready). The latter is useful when connecting to blades.
|
|
"""
|
|
SSH to a blade from the enc takes about 6-7s. If we are at 15s, something is probably wrong. connect servers fail due to:
|
|
a) There are too many users on SSH (ungraceful sessions?)
|
|
b) The iLO is unavailable (has been just reset)
|
|
c) The blade has been just plugged in (not initialized)
|
|
d) SSH access has been disabled
|
|
e) other
|
|
All but e) are known to drop you back in to the enc's shell, in what case we'll be in a problematic spot, since we're going to exit out of the enc ssh, not the blade's.
|
|
#TODO: somehow magically detect if we've successfully connected to the blade. There's no standard way to determine.
|
|
The probable way would be to look at the prompt. Blade's seem to all be '</>hpiLO-> '.
|
|
I would fetch the prompt on connection to the enc, then compare the current prompt to determine if we're connected to a blade.
|
|
"""
|
|
lockedoutCounter += 1
|
|
if lockedoutCounter > 30:
|
|
raise EOFError("Waited for a prompt for 15s. Something is probably wrong. Aborting.")
|
|
else:
|
|
logging.info("No prompt, waiting for more data.")
|
|
sleep(0.5)
|
|
cmdout += channel.recv(65536).decode("ascii") # Get output, capped to 64KiB, format ascii. | Must get, otherwise next command will get this command's output.
|
|
#breakpoint()
|
|
|
|
cmdout = cmdout.split('\n')[2:-3] # Split to a list, per newlines. Remove first 2, and last 3 lines.
|
|
#cmdout = '\n'.join(cmdout) # DO NOT UNCOMMENT THIS, HERE FOR ONLY REFERENCE TO JOIN THE LINES BACK.
|
|
return(cmdout) # Return list of output stuff.
|
|
|
|
### MAIN ###
|
|
|
|
logging.debug(ssh_runcmd('show date')) # Get rid of motd, init for next cmds. This is better than indefinitely reading buffer before any command as to counter this.
|
|
|
|
|
|
## Get list of blades with some added info ##
|
|
# TESTING DATA IN USE
|
|
# serverName = ssh_runcmd('show server names')
|
|
serverName = ['Bay Server Name Serial Number Status Power UID Partner\r', '--- ------------------------------------------------- --------------- -------- ------- --- -------\r', ' 1 tty-lab-1 OK On Off \r', ' 2 tty-lab-2 CZ320263P9 OK On Off \r', ' 3 tty-lab-3 CZJ14410KP Failed On Off \r', ' 4 kspve1 CZJ14410KK OK On Off \r', ' 5 kspve2-2 OK On Off \r', ' 6 kspve3 OK On Off \r', ' 7 plaes-blade OK On Off \r', ' 8 Ringly-01 CZ3402Y48C OK On Off \r', ' 9 toomas-lepik CZ3217FNYE OK On Off \r', ' 10 toomas-lepik2 CZ3217FFSS OK On Off \r', ' 11 [Absent] \r', ' 12 erki-naumanis OK On Off \r', ' 13 [Absent] \r', ' 14 [Absent] \r', ' 15 [Absent] \r', ' 16 [Absent] \r']
|
|
servers = listServers.listServers(serverName)
|
|
|
|
baysInUse = [x[0] for x in servers] # List of blades in use.
|
|
baysInUseCount = len(baysInUse) # How many bays in use.
|
|
logging.info("There are " + str(baysInUseCount+1) + "servers presenet.") # Further optimizations could be made by not connecting to servers, what are turned off.
|
|
|
|
|
|
## Get blade data ##
|
|
for n in range(baysInUseCount):
|
|
bay = servers[n][0] # We want the bay, not how many times we have looped over.
|
|
logging.info("Accessing server " + str(bay))
|
|
logging.debug( ssh_runcmd('connect server ' + str(bay)) ) # Use the enc as a jump host.
|
|
powerInfoTmp = ssh_runcmd('show system1/oemhp_power1') # Get the data
|
|
|
|
presentPower = [i for i in powerInfoTmp if i.startswith(' oemhp_PresentPower=')][0][23:-7] # Get the line with PresentPower, then remove first 23, and last 7 chars to end up with the Watts DC the blade is directly using.
|
|
servers[n].append(int(presentPower)) # And push it to our miniDB of the servers list.
|
|
logging.info("UsageRawDC: " + presentPower)
|
|
|
|
logging.debug(ssh_runcmd('exit')) # Exit the blade's iLO, return to the enc's iLO SSH interface.
|
|
#breakpoint()
|
|
|
|
#print(servers)
|
|
#breakpoint()
|
|
# Fields: [BayNumber "Server Name", "Serial Number", "Status", "Power", "UID Partner", presentPowerDCDirect]
|
|
|
|
|
|
## Get enc's _AC_ usage. ##
|
|
encPowerUsageAC = int(str.strip( [i for i in ssh_runcmd('show power') if i.startswith('Present Power:')][0][14:-9] ))
|
|
logging.info("Server total usage AC: " + str(encPowerUsageAC))
|
|
|
|
## End sesion with enc. ##
|
|
channel.send('exit' + '\n') # Using lower level cmd since we're not going to get a prompt back here.
|
|
ssh.close()
|
|
|
|
|
|
## Calculating blade usage by percentage. ##
|
|
# Keep in mind the querying of the data took a while, a minute or so.
|
|
relativeUsage.relativeUsage(encPowerUsageAC, baysInUseCount, servers)
|
|
|
|
print(servers)
|
|
breakpoint()
|
|
print()
|