squeue_stats.py

Peter Amstutz, 08/03/2018 04:51 PM

 
1
import subprocess
2
import datetime
3
import pprint
4

    
5
sq = subprocess.check_output(["squeue", "--noheader", "--format=%c|%m|%d|%r|%j|%f|%Q|%T|%V"])
6

    
7
busy = 0
8
pending = 0
9
total = 0
10
pendingreason = {}
11
pendingfeatures = {}
12

    
13
now = datetime.datetime.now()
14

    
15
ages = []
16

    
17
first = True
18
head_of_line = {}
19

    
20
for n in sq.splitlines():
21
        cores, memory, disk, reason, jobname, features, priority, state, subtime = n.split("|")
22
        
23
        age = now - datetime.datetime.strptime(subtime, "%Y-%m-%dT%H:%M:%S")
24
        if state == "RUNNING":
25
                busy += 1
26
        else:
27
                if first:
28
                        head_of_line["reason"] = reason
29
                        head_of_line["jobname"] = jobname
30
                        head_of_line["features"] = features
31
                        head_of_line["priority"] = priority
32
                        head_of_line["age"] = str(age)
33
                        first = False
34
                pending += 1
35
                ages.append(age)
36
        if reason != "None":
37
                pendingreason.setdefault(reason, 0)
38
                pendingreason[reason] += 1
39
                pendingfeatures.setdefault(features, 0)
40
                pendingfeatures[features] += 1
41

    
42
        total += 1
43

    
44
sinfo_out = subprocess.check_output(["sinfo", "--noheader", "--sort=-p,v", "--format=%n|%t|%f"])
45
totalnodes = 0
46
nodestates = {}
47
nodefeatures = {}
48

    
49
for n in sinfo_out.splitlines():
50
        name, state, features = n.split("|")
51
        if state.endswith("*"):
52
                continue
53
        totalnodes += 1
54
        nodestates.setdefault(state, 0)
55
        nodestates[state] += 1
56

    
57
        if state == "idle":
58
                nodefeatures.setdefault(features, 0)
59
                nodefeatures[features] += 1
60

    
61

    
62
agesum = datetime.timedelta()
63
agemax = datetime.timedelta()
64
agemin = datetime.timedelta(1000)
65
for a in ages:
66
        agesum += a
67
        if a > agemax:
68
                agemax = a
69
        if a < agemin:
70
                agemin = a
71

    
72
b = {}
73
b["running"] = busy
74
b["pending"] = pending
75
b["total"] = total
76
b["pendingreason"] = pendingreason
77
b["pendingfeatures"] = pendingfeatures
78
b["ages"] = {
79
        "min": str(agemin),
80
        "max": str(agemax),
81
        "avg": str(agesum / len(ages))
82
}
83
b["head_of_line"] = head_of_line
84
b["totalnodes"] = totalnodes
85
b["nodestates"] = nodestates
86
b["nodefeatures"] = nodefeatures
87

    
88
pprint.pprint(b)