Arvados

#!/usr/bin/env python3

# arv container_request list --filters '[["properties", "exists", "template_uuid"], 

# ["container.state", "=", "Complete"], ["container.exit_code", "=", 0]]' --select '["name", "uuid", "cumulative_cost"]'

import arvados

import arvados.util

import os

import json

import csv

api = arvados.api()

by_workflow = {}

if os.path.exists("costs.txt"):

    with open("costs.txt", "rt") as f:

        by_workflow = json.load(f)

if not by_workflow:

    for item in arvados.util.keyset_list_all(api.container_requests().list,

      filters=[["properties", "exists", "template_uuid"], ["container.state", "=", "Complete"], ["container.exit_code", "=", 0]],

      select=["name", "uuid", "properties", "cumulative_cost", "created_at"]):

        template_uuid = item["properties"]["template_uuid"]

        by_workflow.setdefault(template_uuid, [])

        by_workflow[template_uuid].append(item)

        print(item)

with open("costs.txt", "wt") as f:

    json.dump(by_workflow, f)

costs_out = open("costs.csv", "wt")

costs_csv = csv.writer(costs_out)

costs_csv.writerow(("uuid", "name", "count", "total", "average"))

for wf in by_workflow:

    try:

        wfitem = api.workflows().get(uuid=wf).execute()

    except:

        wfitem = {"uuid": wf, "name": ""}

    count = 0

    total = 0

    for cr in by_workflow[wf]:

       cost = cr["cumulative_cost"]

       if cost == 0:

           continue

       total += cost

       count += 1

    costs_csv.writerow((wfitem["uuid"], wfitem["name"], count, total, total/count if count > 0 else 0))