+ export ARVADOS_API_TOKEN=systemroottokenmushaveatleast32characters + ARVADOS_API_TOKEN=systemroottokenmushaveatleast32characters + export ARVADOS_API_HOST=harpo.local:8443 + ARVADOS_API_HOST=harpo.local:8443 + export ARVADOS_API_HOST_INSECURE=true + ARVADOS_API_HOST_INSECURE=true + set -o pipefail + curl -s -o /dev/null 'https://harpo.local:8443/users/welcome?return_to=%2F' + echo 'Creating Arvados Standard Docker Images project' Creating Arvados Standard Docker Images project ++ cut -d- -f1 ++ arv --format=uuid user current + uuid_prefix=harpo ++ arv --format=uuid group list --filters '[["name", "=", "Arvados Standard Docker Images"]]' + project_uuid=harpo-j7d0g-z21k9p47jqw0e18 + '[' xharpo-j7d0g-z21k9p47jqw0e18 = x ']' + echo 'Arvados project uuid is '\''harpo-j7d0g-z21k9p47jqw0e18'\''' Arvados project uuid is 'harpo-j7d0g-z21k9p47jqw0e18' + echo 'Uploading arvados/jobs'\'' docker image to the project' Uploading arvados/jobs' docker image to the project + VERSION=2.1.1 + arv-keepdocker --pull arvados/jobs 2.1.1 --project-uuid harpo-j7d0g-z21k9p47jqw0e18 2.1.1: Pulling from arvados/jobs Digest: sha256:c422fde229402b1c291be15d327664dca54ad6d0217761ce1531f43e08218b38 Status: Image is up to date for arvados/jobs:2.1.1 docker.io/arvados/jobs:2.1.1 harpo-4zz18-7xvbpw7gvf3viyn + echo 'Creating initial user '\''admin'\''' Creating initial user 'admin' ++ arv --format=uuid user list --filters '[["email", "=", "admin@harpo.local"], ["username", "=", "admin"]]' + user_uuid=harpo-tpzed-xb7ghdpgz1sav5v + '[' xharpo-tpzed-xb7ghdpgz1sav5v = x ']' + echo 'Activating user '\''admin'\''' Activating user 'admin' + arv user update --uuid harpo-tpzed-xb7ghdpgz1sav5v --user '{"is_active": true}' { "created_at":"2021-10-08T09:14:48.069151000Z", "email":"admin@harpo.local", "etag":"9bkyxpstfzt88hm14zf5jjo6j", "first_name":"", "full_name":"", "identity_url":"", "is_active":true, "is_admin":true, "is_invited":true, "kind":"arvados#user", "last_name":"", "modified_at":"2021-10-08T09:14:48.069732000Z", "modified_by_client_uuid":"harpo-ozdt8-r3rh8t1j1wooyif", "modified_by_user_uuid":"harpo-tpzed-000000000000000", "owner_uuid":"harpo-tpzed-000000000000000", "prefs":{}, "username":"admin", "uuid":"harpo-tpzed-xb7ghdpgz1sav5v", "writable_by":[ "harpo-tpzed-000000000000000", "harpo-tpzed-000000000000000", "harpo-j7d0g-000000000000000" ] } + echo 'Getting the user API TOKEN' Getting the user API TOKEN ++ arv api_client_authorization list ++ head -1 ++ jq -r '.items[] | select( .owner_uuid == "harpo-tpzed-xb7ghdpgz1sav5v" ).api_token' + user_api_token=3sukm1isfoq6widdi6dauhyith61ih2dnsn4k4686didxttbqp + '[' x3sukm1isfoq6widdi6dauhyith61ih2dnsn4k4686didxttbqp = x ']' + echo 'API TOKEN FOR user '\''admin'\'': '\''3sukm1isfoq6widdi6dauhyith61ih2dnsn4k4686didxttbqp'\''.' API TOKEN FOR user 'admin': '3sukm1isfoq6widdi6dauhyith61ih2dnsn4k4686didxttbqp'. + echo 'Switching to user '\''admin'\''' Switching to user 'admin' + export ARVADOS_API_TOKEN=3sukm1isfoq6widdi6dauhyith61ih2dnsn4k4686didxttbqp + ARVADOS_API_TOKEN=3sukm1isfoq6widdi6dauhyith61ih2dnsn4k4686didxttbqp + echo 'Running test CWL workflow' Running test CWL workflow + cwl-runner --debug hasher-workflow.cwl hasher-workflow-job.yml INFO /usr/bin/cwl-runner 2.3.0.dev20211006204117, arvados-python-client 2.3.0.dev20211006204117, cwltool 3.1.20210922203925 INFO Resolved 'hasher-workflow.cwl' to 'file:///tmp/cluster_tests/hasher-workflow.cwl' INFO hasher-workflow.cwl:34:7: Unknown hint http://arvados.org/cwl#IntermediateOutput INFO hasher-workflow.cwl:36:7: Unknown hint WorkReuse INFO hasher-workflow.cwl:48:7: Unknown hint http://arvados.org/cwl#IntermediateOutput INFO hasher-workflow.cwl:50:7: Unknown hint WorkReuse INFO hasher-workflow.cwl:62:7: Unknown hint http://arvados.org/cwl#IntermediateOutput INFO hasher-workflow.cwl:64:7: Unknown hint WorkReuse INFO Using cluster harpo (https://workbench2.harpo.local:8443/) INFO Upload local files: "test.txt" DEBUG {'harpo-bi6l4-8cf94edb2dd554bd': OrderedDict([('href', '/keep_services/harpo-bi6l4-8cf94edb2dd554bd'), ('kind', 'arvados#keepService'), ('etag', '8qis2fstf5tpnye49t7ip7x48'), ('uuid', 'harpo-bi6l4-8cf94edb2dd554bd'), ('owner_uuid', 'harpo-tpzed-000000000000000'), ('created_at', '2021-10-08T09:14:00.553391000Z'), ('modified_by_client_uuid', None), ('modified_by_user_uuid', 'harpo-tpzed-000000000000000'), ('modified_at', '2021-10-08T09:14:00.553391000Z'), ('service_host', 'keep0.harpo.local'), ('service_port', 25107), ('service_ssl_flag', False), ('service_type', 'disk'), ('read_only', False), ('_service_root', 'http://keep0.harpo.local:25107/')])} DEBUG 7f2cee57647f15dd443e35537b202981+104: ['http://keep0.harpo.local:25107/'] DEBUG Pool max threads is 2 DEBUG Request: PUT http://keep0.harpo.local:25107/7f2cee57647f15dd443e35537b202981 INFO PUT 200: 104 bytes in 21.840810775756836 msec (0.005 MiB/sec) DEBUG KeepWriterThread succeeded 7f2cee57647f15dd443e35537b202981+104 http://keep0.harpo.local:25107/ INFO Using collection f55e750025853f5b8ccae3ca79240f65+54 (harpo-4zz18-5dz2ijnryheb29y) INFO hasher-workflow.cwl:34:7: Unknown hint http://arvados.org/cwl#IntermediateOutput INFO hasher-workflow.cwl:36:7: Unknown hint http://arvados.org/cwl#ReuseRequirement INFO hasher-workflow.cwl:48:7: Unknown hint http://arvados.org/cwl#IntermediateOutput INFO hasher-workflow.cwl:50:7: Unknown hint http://arvados.org/cwl#ReuseRequirement INFO hasher-workflow.cwl:62:7: Unknown hint http://arvados.org/cwl#IntermediateOutput INFO hasher-workflow.cwl:64:7: Unknown hint http://arvados.org/cwl#ReuseRequirement INFO hasher-workflow.cwl:34:7: Unknown hint http://arvados.org/cwl#IntermediateOutput INFO hasher-workflow.cwl:36:7: Unknown hint http://arvados.org/cwl#ReuseRequirement INFO hasher-workflow.cwl:48:7: Unknown hint http://arvados.org/cwl#IntermediateOutput INFO hasher-workflow.cwl:50:7: Unknown hint http://arvados.org/cwl#ReuseRequirement INFO hasher-workflow.cwl:62:7: Unknown hint http://arvados.org/cwl#IntermediateOutput INFO hasher-workflow.cwl:64:7: Unknown hint http://arvados.org/cwl#ReuseRequirement INFO Using collection cache size 256 MiB DEBUG ENTER jobiter 1633684947.0680757 DEBUG EXIT jobiter 1633684947.0694785 0.0014028549194335938 DEBUG ENTER run 1633684947.0696337 INFO [container hasher-workflow.cwl] submitted container_request harpo-xvhdp-76r9nn4gw6ogur5 INFO Monitor workflow progress at https://workbench2.harpo.local:8443/processes/harpo-xvhdp-76r9nn4gw6ogur5 DEBUG EXIT run 1633684947.269519 0.19988536834716797 DEBUG ENTER jobiter 1633684947.2696536 DEBUG EXIT jobiter 1633684947.2697353 8.177757263183594e-05 INFO [container hasher-workflow.cwl] harpo-xvhdp-76r9nn4gw6ogur5 is Final DEBUG 1bf5ff3c9e60ba29729d6ae60c20d645+23832+A5df697f0456a3ffe6a326c3c996c0770bc8aa45f@617282df: ['http://keep0.harpo.local:25107/'] DEBUG Request: GET http://keep0.harpo.local:25107/1bf5ff3c9e60ba29729d6ae60c20d645+23832+A5df697f0456a3ffe6a326c3c996c0770bc8aa45f@617282df INFO GET 200: 23832 bytes in 2.071857452392578 msec (10.970 MiB/sec) ERROR [container hasher-workflow.cwl] (harpo-dz642-c2t0449e7i9tosy) error log: 2021-10-08T09:22:32.937976160Z stderr conn, authority, uri, request_uri, method, body, headers, redirections, cachekey, 2021-10-08T09:22:32.937976160Z stderr File "/usr/share/python3/dist/python3-arvados-cwl-runner/lib/python3.7/site-packages/httplib2/__init__.py", line 1424, in _request 2021-10-08T09:22:32.937976160Z stderr (response, content) = self._conn_request(conn, request_uri, method, body, headers) 2021-10-08T09:22:32.937976160Z stderr File "/usr/share/python3/dist/python3-arvados-cwl-runner/lib/python3.7/site-packages/httplib2/__init__.py", line 1353, in _conn_request 2021-10-08T09:22:32.937976160Z stderr raise ServerNotFoundError("Unable to find the server at %s" % conn.host) 2021-10-08T09:22:32.937976160Z stderr httplib2.error.ServerNotFoundError: [req-melxbchzhcvwh4c67xeb] Unable to find the server at harpo.local 2021-10-08T09:22:32.937976160Z stderr 2021-10-08T09:22:32.937976160Z stderr During handling of the above exception, another exception occurred: 2021-10-08T09:22:32.937976160Z stderr 2021-10-08T09:22:32.937976160Z stderr Traceback (most recent call last): 2021-10-08T09:22:32.937976160Z stderr File "/usr/share/python3/dist/python3-arvados-cwl-runner/lib/python3.7/site-packages/arvados_cwl/__init__.py", line 298, in main 2021-10-08T09:22:32.937976160Z stderr keep_params={"num_retries": 4}) 2021-10-08T09:22:32.937976160Z stderr File "/usr/share/python3/dist/python3-arvados-cwl-runner/lib/python3.7/site-packages/arvados/safeapi.py", line 33, in __init__ 2021-10-08T09:22:32.937976160Z stderr self.localapi() 2021-10-08T09:22:32.937976160Z stderr File "/usr/share/python3/dist/python3-arvados-cwl-runner/lib/python3.7/site-packages/arvados/safeapi.py", line 40, in localapi 2021-10-08T09:22:32.937976160Z stderr **self.api_params) 2021-10-08T09:22:32.937976160Z stderr File "/usr/share/python3/dist/python3-arvados-cwl-runner/lib/python3.7/site-packages/arvados/api.py", line 293, in api_from_config 2021-10-08T09:22:32.937976160Z stderr return api(version=version, host=host, token=token, insecure=insecure, **kwargs) 2021-10-08T09:22:32.937976160Z stderr File "/usr/share/python3/dist/python3-arvados-cwl-runner/lib/python3.7/site-packages/arvados/api.py", line 251, in api 2021-10-08T09:22:32.937976160Z stderr svc = apiclient_discovery.build('arvados', version, cache_discovery=False, **kwargs) 2021-10-08T09:22:32.937976160Z stderr File "/usr/share/python3/dist/python3-arvados-cwl-runner/lib/python3.7/site-packages/googleapiclient/_helpers.py", line 134, in positional_wrapper 2021-10-08T09:22:32.937976160Z stderr return wrapped(*args, **kwargs) 2021-10-08T09:22:32.937976160Z stderr File "/usr/share/python3/dist/python3-arvados-cwl-runner/lib/python3.7/site-packages/googleapiclient/discovery.py", line 276, in build 2021-10-08T09:22:32.937976160Z stderr num_retries=num_retries, 2021-10-08T09:22:32.937976160Z stderr File "/usr/share/python3/dist/python3-arvados-cwl-runner/lib/python3.7/site-packages/googleapiclient/discovery.py", line 376, in _retrieve_discovery_doc 2021-10-08T09:22:32.937976160Z stderr resp, content = req.execute(num_retries=num_retries) 2021-10-08T09:22:32.937976160Z stderr File "/usr/share/python3/dist/python3-arvados-cwl-runner/lib/python3.7/site-packages/googleapiclient/_helpers.py", line 134, in positional_wrapper 2021-10-08T09:22:32.937976160Z stderr return wrapped(*args, **kwargs) 2021-10-08T09:22:32.937976160Z stderr File "/usr/share/python3/dist/python3-arvados-cwl-runner/lib/python3.7/site-packages/googleapiclient/http.py", line 909, in execute 2021-10-08T09:22:32.937976160Z stderr headers=self.headers, 2021-10-08T09:22:32.937976160Z stderr File "/usr/share/python3/dist/python3-arvados-cwl-runner/lib/python3.7/site-packages/googleapiclient/http.py", line 204, in _retry_request 2021-10-08T09:22:32.937976160Z stderr raise exception 2021-10-08T09:22:32.937976160Z stderr File "/usr/share/python3/dist/python3-arvados-cwl-runner/lib/python3.7/site-packages/googleapiclient/http.py", line 177, in _retry_request 2021-10-08T09:22:32.937976160Z stderr resp, content = http.request(uri, method, *args, **kwargs) 2021-10-08T09:22:32.937976160Z stderr File "/usr/share/python3/dist/python3-arvados-cwl-runner/li 2021-10-08T09:22:32.938052102Z stderr b/python3.7/site-packages/arvados/api.py", line 120, in _intercept_http_request 2021-10-08T09:22:32.938052102Z stderr raise type(e)(*e.args) 2021-10-08T09:22:32.938052102Z stderr httplib2.error.ServerNotFoundError: [req-melxbchzhcvwh4c67xeb] Unable to find the server at harpo.local 2021-10-08T09:22:33.629059065Z crunch-run Container exited with code: 1 2021-10-08T09:22:33.901196089Z crunch-run Complete ERROR Overall process status is permanentFail INFO Final output collection 5551f5d7ec57a8aa9758d14a976e87e9+57 INFO Output at https://workbench2.harpo.local:8443/collections/5551f5d7ec57a8aa9758d14a976e87e9+57 {} WARNING Final process status is permanentFail