Arvados

There were successful API calls:

bzgrep 21:11:4 /var/www/4xphq.arvadosapi.com/current/log/production.log-20170804.bz2 | grep -v '"GET"' | grep -v pipeline_instance

{"method":"PUT","path":"/arvados/v1/job_tasks/4xphq-ot0gb-alnmzg8q7js49jl","format":"json","controller":"Arvados::V1::JobTasksController","action":"update","status":200,"duration":105.2,"view":0.26,"db":94.77,"params":{"output":"40b6a6781b95f09bdcf219b17e9818c9+105+A030c1fcf7484a894974a2d1ff6eb8f1a064d19bd@5996068c","success":true,"progress":1.0,"alt":"json","job_task":{"output":"40b6a6781b95f09bdcf219b17e9818c9+105+A030c1fcf7484a894974a2d1ff6eb8f1a064d19bd@5996068c","progress":1.0,"success":true}},"@timestamp":"2017-08-03T21:11:40.153237768Z","@version":"1","message":"[200] PUT /arvados/v1/job_tasks/4xphq-ot0gb-alnmzg8q7js49jl (Arvados::V1::JobTasksController#update)"}
{"method":"PUT","path":"/arvados/v1/job_tasks/4xphq-ot0gb-alnmzg8q7js49jl","format":"json","controller":"Arvados::V1::JobTasksController","action":"update","status":200,"duration":135.21,"view":0.25,"db":123.65,"params":{"job_task":"{\"output\":\"40b6a6781b95f09bdcf219b17e9818c9+105+A030c1fcf7484a894974a2d1ff6eb8f1a064d19bd@5996068c\",\"owner_uuid\":\"4xphq-tpzed-7cc4bktyfzdcm8f\",\"created_at\":\"2017-08-03T21:11:35.108480000Z\",\"job_uuid\":\"4xphq-8i9sb-mo0iju68fdghirj\",\"modified_by_user_uuid\":\"4xphq-tpzed-7cc4bktyfzdcm8f\",\"modified_at\":\"2017-08-03T21:11:40.051735000Z\",\"parameters\":{\"input\":\"49c5e65830378703643f5d8cbe424567+87/./4xphq-8i9sb-fmwod1qn74cemdp.log.txt\"},\"modified_by_client_uuid\":null,\"created_by_job_task_uuid\":\"4xphq-ot0gb-4y22jc2jm28pxj3\",\"uuid\":\"4xphq-ot0gb-alnmzg8q7js49jl\",\"kind\":\"arvados#jobTask\",\"etag\":\"fgsw161e1yhm3wfs7akrd2ps\",\"finished_at\":\"2017-08-03T21:11:40Z\",\"started_at\":\"2017-08-03T21:11:37.000000000Z\",\"href\":\"/job_tasks/4xphq-ot0gb-alnmzg8q7js49jl\",\"sequence\":1,\"qsequence\":9878,\"progress\":1}"},"@timestamp":"2017-08-03T21:11:40.704703962Z","@version":"1","message":"[200] PUT /arvados/v1/job_tasks/4xphq-ot0gb-alnmzg8q7js49jl (Arvados::V1::JobTasksController#update)"}
{"method":"PUT","path":"/arvados/v1/jobs/4xphq-8i9sb-mo0iju68fdghirj","format":"json","controller":"Arvados::V1::JobsController","action":"update","status":200,"duration":135.74,"view":0.26,"db":112.79,"params":{"job":"{\"tasks_summary\":{\"failed\":0,\"todo\":0,\"running\":0,\"done\":2}}"},"@timestamp":"2017-08-03T21:11:40.914881579Z","@version":"1","message":"[200] PUT /arvados/v1/jobs/4xphq-8i9sb-mo0iju68fdghirj (Arvados::V1::JobsController#update)"}
{"method":"POST","path":"/arvados/v1/collections","format":"json","controller":"Arvados::V1::CollectionsController","action":"create","status":200,"duration":206.97,"view":0.27,"db":196.21,"params":{"manifest_text":". 68cd682060702def3995f1a253ea34d2+71+A6146315ceba5e023d9de8bafea4daab5cf6f116c@5996068b 0:71:md5sum.txt\n","owner_uuid":"4xphq-tpzed-7cc4bktyfzdcm8f","alt":"json","collection":{"owner_uuid":"4xphq-tpzed-7cc4bktyfzdcm8f","manifest_text":". 68cd682060702def3995f1a253ea34d2+71+A6146315ceba5e023d9de8bafea4daab5cf6f116c@5996068b 0:71:md5sum.txt\n"}},"@timestamp":"2017-08-03T21:11:41.733535900Z","@version":"1","message":"[200] POST /arvados/v1/collections (Arvados::V1::CollectionsController#create)"}
{"method":"PUT","path":"/arvados/v1/jobs/4xphq-8i9sb-mo0iju68fdghirj","format":"json","controller":"Arvados::V1::JobsController","action":"update","status":200,"duration":194.86,"view":0.26,"db":170.72,"params":{"job":"{\"output\":\"3b7d583c29a4e2089b7a64d43cdc4077+54\"}"},"@timestamp":"2017-08-03T21:11:41.978907904Z","@version":"1","message":"[200] PUT /arvados/v1/jobs/4xphq-8i9sb-mo0iju68fdghirj (Arvados::V1::JobsController#update)"}

Crash is source:sdk/perl/lib/Arvados/Request.pm#L51

I'm guessing this is the relevant bit of crunch-job, since it does a "collections/create" call after logging "finish":

  my $log_coll = api_call(
    "collections/create", ensure_unique_name => 1, collection => {
      manifest_text => $log_manifest,
      owner_uuid => $Job->{owner_uuid},
      name => sprintf("Log from %s job %s", $Job->{script}, $Job->{uuid}),
    });

debian:8 perl v5.20.2 libjson-perl 2.61-1

$ perl -mJSON -e 'print JSON::encode_json(JSON::true), "\n"'

debian:9 perl v5.24.1 libjson-perl 2.90-1

$ perl -mJSON -e 'print JSON::encode_json(JSON::true), "\n"'
hash- or arrayref expected (not a simple scalar, use allow_nonref to allow this) at -e line 1.

12078-perl-bool-params @ 64b837f87103a59efb6d4dac2b8dfa3c7c40c09a

$ perl -mJSON -e 'print JSON->new->encode(JSON::true), "\n"'
hash- or arrayref expected (not a simple scalar, use allow_nonref to allow this) at -e line 1.
$ perl -mJSON -e 'print JSON->new->allow_nonref->encode(JSON::true), "\n"'
true