Arvados

import subprocess

import datetime

import pprint

sq = subprocess.check_output(["squeue", "--noheader", "--format=%c|%m|%d|%r|%j|%f|%Q|%T|%V"])

busy = 0

pending = 0

total = 0

pendingreason = {}

pendingfeatures = {}

now = datetime.datetime.now()

ages = []

first = True

head_of_line = {}

for n in sq.splitlines():

	cores, memory, disk, reason, jobname, features, priority, state, subtime = n.split("|")

	age = now - datetime.datetime.strptime(subtime, "%Y-%m-%dT%H:%M:%S")

	if state == "RUNNING":

		busy += 1

	else:

		if first:

			head_of_line["reason"] = reason

			head_of_line["jobname"] = jobname

			head_of_line["features"] = features

			head_of_line["priority"] = priority

			head_of_line["age"] = str(age)

			first = False

		pending += 1

		ages.append(age)

	if reason != "None":

		pendingreason.setdefault(reason, 0)

		pendingreason[reason] += 1

		pendingfeatures.setdefault(features, 0)

		pendingfeatures[features] += 1

	total += 1

sinfo_out = subprocess.check_output(["sinfo", "--noheader", "--sort=-p,v", "--format=%n|%t|%f"])

totalnodes = 0

nodestates = {}

nodefeatures = {}

for n in sinfo_out.splitlines():

	name, state, features = n.split("|")

	if state.endswith("*"):

		continue

	totalnodes += 1

	nodestates.setdefault(state, 0)

	nodestates[state] += 1

	if state == "idle":

		nodefeatures.setdefault(features, 0)

		nodefeatures[features] += 1

agesum = datetime.timedelta()

agemax = datetime.timedelta()

agemin = datetime.timedelta(1000)

for a in ages:

	agesum += a

	if a > agemax:

		agemax = a

	if a < agemin:

		agemin = a

b = {}

b["running"] = busy

b["pending"] = pending

b["total"] = total

b["pendingreason"] = pendingreason

b["pendingfeatures"] = pendingfeatures

b["ages"] = {

	"min": str(agemin),

	"max": str(agemax),

	"avg": str(agesum / len(ages))

}

b["head_of_line"] = head_of_line

b["totalnodes"] = totalnodes

b["nodestates"] = nodestates

b["nodefeatures"] = nodefeatures

pprint.pprint(b)