Sep 05 21:06:44 arv-demo arvados-server[18694]: {"PID":18694,"level":"info","msg":"starting","task":"dispatch-cloud","time":"2022-09-05T21:06:44.584131956Z"}
Sep 05 21:07:00 arv-demo arvados-server[18694]: {"PID":18694,"command":"[/var/lib/arvados/bin/arvados-server dispatch-cloud -config /var/lib/arvados/wwwtmp/config.yml]","dir":"/var/lib/arvados/tmp","level":"info","msg":"executing","time":"2022-09-05T21:07:00.365163407Z"}
Sep 05 21:07:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"level":"warning","msg":"deprecated or unknown config entry: SourceSHA256","time":"2022-09-05T21:07:00.563376798Z"}
Sep 05 21:07:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"level":"warning","msg":"deprecated or unknown config entry: SourceTimestamp","time":"2022-09-05T21:07:00.563543156Z"}
Sep 05 21:07:00 arv-demo arvados-server[18694]: {"PID":18694,"level":"info","msg":"waiting","task":"dispatch-cloud","time":"2022-09-05T21:07:00.595675664Z"}
Sep 05 21:07:00 arv-demo arvados-server[18694]: {"PID":18694,"level":"info","msg":"ready","task":"dispatch-cloud","time":"2022-09-05T21:07:00.595692231Z"}
Sep 05 21:07:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"level":"error","msg":"error updating queue: request failed: https://arva3.di-pentima.com.ar:4440/arvados/v1/api_client_authorizations/current: 502 Bad Gateway","time":"2022-09-05T21:07:00.659025703Z"}
Sep 05 21:07:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"level":"info","msg":"waiting 1s before retry","time":"2022-09-05T21:07:00.659068889Z"}
Sep 05 21:07:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","N":0,"PID":18918,"level":"info","msg":"loaded initial instance list","time":"2022-09-05T21:07:00.757135488Z"}
Sep 05 21:07:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","Listen":"[::]:9006","PID":18918,"Service":"arvados-dispatch-cloud","URL":"http://0.0.0.0:9006/","Version":"2.1.0-2866-g10440ac12 (go1.17.7)","level":"info","msg":"listening","time":"2022-09-05T21:07:00.757288097Z"}
Sep 05 21:07:01 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-46ng47b63nwr18rjcai2","level":"info","msg":"request","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:07:01.317922327Z"}
Sep 05 21:07:01 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-46ng47b63nwr18rjcai2","level":"info","msg":"response","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:07:01.317995442Z","timeToStatus":0.000047,"timeTotal":0.000050,"timeWriteBody":0.000003}
Sep 05 21:07:02 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-vqo6lzdxbq7b1i2w3yq9","level":"info","msg":"request","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:07:02.315921673Z"}
Sep 05 21:07:02 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-vqo6lzdxbq7b1i2w3yq9","level":"info","msg":"response","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:07:02.316004985Z","timeToStatus":0.000072,"timeTotal":0.000076,"timeWriteBody":0.000003}
Sep 05 21:07:06 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-5hp1k53wysc1l7mkywlz","level":"info","msg":"request","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:07:06.320717785Z"}
Sep 05 21:07:06 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-5hp1k53wysc1l7mkywlz","level":"info","msg":"response","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:07:06.320786291Z","timeToStatus":0.000057,"timeTotal":0.000063,"timeWriteBody":0.000006}
Sep 05 21:07:06 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"level":"info","msg":"FixStaleLocks starting.","time":"2022-09-05T21:07:06.876306400Z"}
Sep 05 21:07:06 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"level":"info","msg":"FixStaleLocks finished (44.205µs), starting scheduling.","time":"2022-09-05T21:07:06.876351186Z"}
Sep 05 21:07:07 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-20uccx8khav1qrzbloem","level":"info","msg":"request","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:07:07.326096156Z"}
Sep 05 21:07:07 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-20uccx8khav1qrzbloem","level":"info","msg":"response","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:07:07.326243837Z","timeToStatus":0.000136,"timeTotal":0.000140,"timeWriteBody":0.000004}
Sep 05 21:07:08 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-1pm4pjns58e6b183ce6g","level":"info","msg":"request","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:07:08.315649572Z"}
Sep 05 21:07:08 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","PID":18918,"RequestID":"req-1pm4pjns58e6b183ce6g","level":"info","msg":"response","remoteAddr":"127.0.0.1:35052","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:07:08.315734964Z","timeToStatus":0.000078,"timeTotal":0.000081,"timeWriteBody":0.000004}
Sep 05 21:08:16 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-5178f8vs4i6ai6o","InstanceType":"localhost","PID":18918,"Priority":562948291009222799,"State":"Queued","level":"info","msg":"adding container to queue","time":"2022-09-05T21:08:16.951020182Z"}
Sep 05 21:08:17 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-5178f8vs4i6ai6o","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:08:17.031358511Z"}
Sep 05 21:08:17 arv-demo arvados-server[18694]: [dispatch-cloud] {"Address":"","ClusterID":"arva3","IdleBehavior":"run","Instance":"localhost","InstanceType":"localhost","PID":18918,"State":"booting","level":"info","msg":"instance appeared in cloud","time":"2022-09-05T21:08:17.119222371Z"}
Sep 05 21:15:26 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-5178f8vs4i6ai6o","PID":18918,"Priority":0,"State":"Locked","level":"info","msg":"requeueing locked container because priority=0","time":"2022-09-05T21:15:26.937755675Z"}
Sep 05 21:15:27 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-5178f8vs4i6ai6o","PID":18918,"Priority":0,"State":"Queued","level":"info","msg":"container on hold -- dropping from queue","time":"2022-09-05T21:15:27.013736444Z"}
Sep 05 21:15:27 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-5178f8vs4i6ai6o","PID":18918,"State":"Queued","level":"info","msg":"dropping container from queue","time":"2022-09-05T21:15:27.013789633Z"}
Sep 05 21:15:46 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"Priority":562948291008766280,"State":"Queued","level":"info","msg":"adding container to queue","time":"2022-09-05T21:15:46.943927315Z"}
Sep 05 21:18:20 arv-demo arvados-server[18694]: [dispatch-cloud] {"Address":"","ClusterID":"arva3","Duration":603671548445,"Instance":"localhost","InstanceType":"localhost","PID":18918,"Since":"2022-09-05T21:08:17.119435471Z","State":"booting","level":"warning","msg":"new instance unresponsive, shutting down","time":"2022-09-05T21:18:20.798336720Z"}
Sep 05 21:18:20 arv-demo arvados-server[18694]: [dispatch-cloud] {"Address":"","ClusterID":"arva3","Duration":603671548445,"Instance":"localhost","InstanceType":"localhost","PID":18918,"ProbeStart":"2022-09-05T21:18:20.790983922Z","level":"info","msg":"boot failed","stderr":"","time":"2022-09-05T21:18:20.798447965Z"}
Sep 05 21:18:20 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:18:20.798513056Z"}
Sep 05 21:18:20 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:18:20.928607177Z"}
Sep 05 21:18:20 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":18918,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:18:20.978498952Z"}
Sep 05 21:19:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":18918,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:19:00.765191191Z"}
Sep 05 21:19:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:19:21.063489911Z"}
Sep 05 21:19:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:19:21.084653683Z"}
Sep 05 21:19:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":18918,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:19:21.084750987Z"}
Sep 05 21:20:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":18918,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:20:00.766247298Z"}
Sep 05 21:20:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:20:21.149823831Z"}
Sep 05 21:20:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:20:21.192124593Z"}
Sep 05 21:20:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":18918,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:20:21.192245708Z"}
Sep 05 21:21:00 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":18918,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:21:00.766925995Z"}
Sep 05 21:21:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:21:21.259514320Z"}
Sep 05 21:21:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","InstanceType":"localhost","PID":18918,"level":"info","msg":"creating new instance","time":"2022-09-05T21:21:21.308168867Z"}
Sep 05 21:21:21 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":18918,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:21:21.308354095Z"}
Sep 05 21:21:46 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","PID":18918,"Priority":0,"State":"Queued","level":"info","msg":"container on hold -- dropping from queue","time":"2022-09-05T21:21:46.957838156Z"}
Sep 05 21:21:46 arv-demo arvados-server[18694]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-xrbibfdknei88ys","PID":18918,"State":"Queued","level":"info","msg":"dropping container from queue","time":"2022-09-05T21:21:46.957877222Z"}
Sep 05 21:21:58 arv-demo arvados-server[22391]: {"PID":22391,"level":"info","msg":"starting","task":"dispatch-cloud","time":"2022-09-05T21:21:58.811163032Z"}
Sep 05 21:21:58 arv-demo arvados-server[22391]: {"PID":22391,"command":"[/var/lib/arvados/bin/arvados-server dispatch-cloud -config /var/lib/arvados/wwwtmp/config.yml]","dir":"/var/lib/arvados/tmp","level":"info","msg":"executing","time":"2022-09-05T21:21:58.890516378Z"}
Sep 05 21:21:58 arv-demo arvados-server[22391]: {"PID":22391,"level":"info","msg":"waiting","task":"dispatch-cloud","time":"2022-09-05T21:21:58.952617406Z"}
Sep 05 21:21:58 arv-demo arvados-server[22391]: {"PID":22391,"level":"info","msg":"ready","task":"dispatch-cloud","time":"2022-09-05T21:21:58.952662731Z"}
Sep 05 21:21:59 arv-demo arvados-server[22391]: [dispatch-cloud] {"level":"warning","msg":"deprecated or unknown config entry: SourceTimestamp","time":"2022-09-05T21:21:59.071738355Z"}
Sep 05 21:21:59 arv-demo arvados-server[22391]: [dispatch-cloud] {"level":"warning","msg":"deprecated or unknown config entry: SourceSHA256","time":"2022-09-05T21:21:59.071993162Z"}
Sep 05 21:21:59 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"level":"error","msg":"error updating queue: request failed: https://arva3.di-pentima.com.ar:4440/arvados/v1/api_client_authorizations/current: 500 Internal Server Error: Get \"//railsapi.internal/arvados/v1/api_client_authorizations/current\": dial tcp 0.0.0.0:9001: connect: connection refused","time":"2022-09-05T21:21:59.163531971Z"}
Sep 05 21:21:59 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"level":"info","msg":"waiting 1s before retry","time":"2022-09-05T21:21:59.163588666Z"}
Sep 05 21:21:59 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","N":0,"PID":22503,"level":"info","msg":"loaded initial instance list","time":"2022-09-05T21:21:59.229689776Z"}
Sep 05 21:21:59 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","Listen":"[::]:9006","PID":22503,"Service":"arvados-dispatch-cloud","URL":"http://0.0.0.0:9006/","Version":"2.1.0-2866-g10440ac12 (go1.17.7)","level":"info","msg":"listening","time":"2022-09-05T21:21:59.231077905Z"}
Sep 05 21:21:59 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-jwgqm4myr8pnqx1aek5a","level":"info","msg":"request","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:21:59.548510077Z"}
Sep 05 21:21:59 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-jwgqm4myr8pnqx1aek5a","level":"info","msg":"response","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:21:59.548622979Z","timeToStatus":0.000103,"timeTotal":0.000105,"timeWriteBody":0.000002}
Sep 05 21:22:00 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-1ptvtvsf62jxm1mb884q","level":"info","msg":"request","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:22:00.541776580Z"}
Sep 05 21:22:00 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-1ptvtvsf62jxm1mb884q","level":"info","msg":"response","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:22:00.541851545Z","timeToStatus":0.000066,"timeTotal":0.000069,"timeWriteBody":0.000003}
Sep 05 21:22:04 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-1l4mvzi3pyj7p6o76f8x","level":"info","msg":"request","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:22:04.549349097Z"}
Sep 05 21:22:04 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-1l4mvzi3pyj7p6o76f8x","level":"info","msg":"response","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:22:04.549680702Z","timeToStatus":0.000086,"timeTotal":0.000089,"timeWriteBody":0.000003}
Sep 05 21:22:05 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"level":"info","msg":"FixStaleLocks starting.","time":"2022-09-05T21:22:05.114120250Z"}
Sep 05 21:22:05 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"level":"info","msg":"FixStaleLocks finished (65.77µs), starting scheduling.","time":"2022-09-05T21:22:05.114183077Z"}
Sep 05 21:22:05 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-1sanbqph43nbw7njaboq","level":"info","msg":"request","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:22:05.542217118Z"}
Sep 05 21:22:05 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-1sanbqph43nbw7njaboq","level":"info","msg":"response","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:22:05.542270689Z","timeToStatus":0.000046,"timeTotal":0.000048,"timeWriteBody":0.000002}
Sep 05 21:22:06 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-1jp2mdrbs7wwc1xta91v","level":"info","msg":"request","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:22:06.543357858Z"}
Sep 05 21:22:06 arv-demo arvados-server[22391]: [dispatch-cloud] {"ClusterID":"arva3","PID":22503,"RequestID":"req-1jp2mdrbs7wwc1xta91v","level":"info","msg":"response","remoteAddr":"127.0.0.1:38822","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:22:06.543417908Z","timeToStatus":0.000057,"timeTotal":0.000059,"timeWriteBody":0.000002}
Sep 05 21:25:49 arv-demo arvados-server[23258]: {"PID":23258,"level":"info","msg":"starting","task":"dispatch-cloud","time":"2022-09-05T21:25:49.074531616Z"}
Sep 05 21:25:58 arv-demo arvados-server[23258]: {"PID":23258,"command":"[/var/lib/arvados/bin/arvados-server dispatch-cloud -config /var/lib/arvados/wwwtmp/config.yml]","dir":"/var/lib/arvados/tmp","level":"info","msg":"executing","time":"2022-09-05T21:25:58.565875498Z"}
Sep 05 21:25:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"level":"warning","msg":"deprecated or unknown config entry: SourceSHA256","time":"2022-09-05T21:25:58.698955888Z"}
Sep 05 21:25:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"level":"warning","msg":"deprecated or unknown config entry: SourceTimestamp","time":"2022-09-05T21:25:58.699610849Z"}
Sep 05 21:25:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"level":"error","msg":"error updating queue: request failed: https://arva3.di-pentima.com.ar:4440/arvados/v1/api_client_authorizations/current: 502 Bad Gateway","time":"2022-09-05T21:25:58.768320924Z"}
Sep 05 21:25:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"level":"info","msg":"waiting 1s before retry","time":"2022-09-05T21:25:58.768363871Z"}
Sep 05 21:25:58 arv-demo arvados-server[23258]: {"PID":23258,"level":"info","msg":"waiting","task":"dispatch-cloud","time":"2022-09-05T21:25:58.775825031Z"}
Sep 05 21:25:58 arv-demo arvados-server[23258]: {"PID":23258,"level":"info","msg":"ready","task":"dispatch-cloud","time":"2022-09-05T21:25:58.775980480Z"}
Sep 05 21:25:58 arv-demo arvados-server[23258]: {"PID":23258,"level":"info","msg":"waiting","targets":"keepproxy+http://0.0.0.0:9007/_health/ping arvados-api-server+http://0.0.0.0:9001/_health/ping arvados-dispatch-cloud+http://0.0.0.0:9006/_health/ping","time":"2022-09-05T21:25:58.844972005Z"}
Sep 05 21:25:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","N":0,"PID":23463,"level":"info","msg":"loaded initial instance list","time":"2022-09-05T21:25:58.867033756Z"}
Sep 05 21:25:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Listen":"[::]:9006","PID":23463,"Service":"arvados-dispatch-cloud","URL":"http://0.0.0.0:9006/","Version":"2.1.0-2866-g10440ac12 (go1.17.7)","level":"info","msg":"listening","time":"2022-09-05T21:25:58.868291411Z"}
Sep 05 21:25:59 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"RequestID":"req-teg395swgg421xh5wox3","level":"info","msg":"request","remoteAddr":"127.0.0.1:39076","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:25:59.811110787Z"}
Sep 05 21:25:59 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"RequestID":"req-teg395swgg421xh5wox3","level":"info","msg":"response","remoteAddr":"127.0.0.1:39076","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:25:59.811194259Z","timeToStatus":0.000075,"timeTotal":0.000077,"timeWriteBody":0.000002}
Sep 05 21:26:03 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"RequestID":"req-1xrw7ponex0pl1jaussq","level":"info","msg":"request","remoteAddr":"127.0.0.1:39076","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:26:03.813513266Z"}
Sep 05 21:26:03 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"RequestID":"req-1xrw7ponex0pl1jaussq","level":"info","msg":"response","remoteAddr":"127.0.0.1:39076","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:26:03.813604953Z","timeToStatus":0.000065,"timeTotal":0.000069,"timeWriteBody":0.000003}
Sep 05 21:26:04 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"level":"info","msg":"FixStaleLocks starting.","time":"2022-09-05T21:26:04.701494864Z"}
Sep 05 21:26:04 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"level":"info","msg":"FixStaleLocks finished (40.255µs), starting scheduling.","time":"2022-09-05T21:26:04.701532766Z"}
Sep 05 21:26:04 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"RequestID":"req-12vzub46fxf8v1xb0frg","level":"info","msg":"request","remoteAddr":"127.0.0.1:39076","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:26:04.809979602Z"}
Sep 05 21:26:04 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"RequestID":"req-12vzub46fxf8v1xb0frg","level":"info","msg":"response","remoteAddr":"127.0.0.1:39076","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:26:04.810084360Z","timeToStatus":0.000096,"timeTotal":0.000099,"timeWriteBody":0.000004}
Sep 05 21:26:05 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"RequestID":"req-yet3fxp6uuk91vujq2l9","level":"info","msg":"request","remoteAddr":"127.0.0.1:39076","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","time":"2022-09-05T21:26:05.810003244Z"}
Sep 05 21:26:05 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","PID":23463,"RequestID":"req-yet3fxp6uuk91vujq2l9","level":"info","msg":"response","remoteAddr":"127.0.0.1:39076","reqBytes":0,"reqForwardedFor":"","reqHost":"0.0.0.0:9006","reqMethod":"GET","reqPath":"_health/ping","reqQuery":"","respBytes":16,"respStatus":"OK","respStatusCode":200,"time":"2022-09-05T21:26:05.810089943Z","timeToStatus":0.000060,"timeTotal":0.000063,"timeWriteBody":0.000003}
Sep 05 21:27:04 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"Priority":562948291008091902,"State":"Queued","level":"info","msg":"adding container to queue","time":"2022-09-05T21:27:04.765740214Z"}
Sep 05 21:27:04 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:27:04.835338227Z"}
Sep 05 21:27:04 arv-demo arvados-server[23258]: [dispatch-cloud] {"Address":"","ClusterID":"arva3","IdleBehavior":"run","Instance":"localhost","InstanceType":"localhost","PID":23463,"State":"booting","level":"info","msg":"instance appeared in cloud","time":"2022-09-05T21:27:04.858262134Z"}
Sep 05 21:37:08 arv-demo arvados-server[23258]: [dispatch-cloud] {"Address":"","ClusterID":"arva3","Duration":604112469667,"Instance":"localhost","InstanceType":"localhost","PID":23463,"Since":"2022-09-05T21:27:04.858354147Z","State":"booting","level":"warning","msg":"new instance unresponsive, shutting down","time":"2022-09-05T21:37:08.977753431Z"}
Sep 05 21:37:08 arv-demo arvados-server[23258]: [dispatch-cloud] {"Address":"","ClusterID":"arva3","Duration":604112469667,"Instance":"localhost","InstanceType":"localhost","PID":23463,"ProbeStart":"2022-09-05T21:37:08.970823821Z","level":"info","msg":"boot failed","stderr":"","time":"2022-09-05T21:37:08.977850147Z"}
Sep 05 21:37:08 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:37:08.977937998Z"}
Sep 05 21:37:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:37:09.040512748Z"}
Sep 05 21:37:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:37:09.113587800Z"}
Sep 05 21:37:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:37:58.873608854Z"}
Sep 05 21:38:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:38:09.186182290Z"}
Sep 05 21:38:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:38:09.220550715Z"}
Sep 05 21:38:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:38:09.220672443Z"}
Sep 05 21:38:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:38:58.874888645Z"}
Sep 05 21:39:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:39:09.292512013Z"}
Sep 05 21:39:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:39:09.317574018Z"}
Sep 05 21:39:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:39:09.317708137Z"}
Sep 05 21:39:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:39:58.876026877Z"}
Sep 05 21:40:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:40:09.396941284Z"}
Sep 05 21:40:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:40:09.429551715Z"}
Sep 05 21:40:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:40:09.429703846Z"}
Sep 05 21:40:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:40:58.876880506Z"}
Sep 05 21:41:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:41:09.514528381Z"}
Sep 05 21:41:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:41:09.583702273Z"}
Sep 05 21:41:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:41:09.612859836Z"}
Sep 05 21:41:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","PID":23463,"State":"Cancelled","level":"info","msg":"container finished -- dropping from queue","time":"2022-09-05T21:41:09.852550279Z"}
Sep 05 21:41:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-iddcpu78ykoz66z","PID":23463,"State":"Cancelled","level":"info","msg":"dropping container from queue","time":"2022-09-05T21:41:09.852598345Z"}
Sep 05 21:41:14 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"Priority":562948291008091902,"State":"Queued","level":"info","msg":"adding container to queue","time":"2022-09-05T21:41:14.788757552Z"}
Sep 05 21:41:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:41:58.877483495Z"}
Sep 05 21:42:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:42:09.701242569Z"}
Sep 05 21:42:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:42:09.743228835Z"}
Sep 05 21:42:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:42:09.743342744Z"}
Sep 05 21:42:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:42:58.878066903Z"}
Sep 05 21:43:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:43:09.829092644Z"}
Sep 05 21:43:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:43:09.890649505Z"}
Sep 05 21:43:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:43:09.913470801Z"}
Sep 05 21:43:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:43:58.879069982Z"}
Sep 05 21:44:09 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:44:09.982321967Z"}
Sep 05 21:44:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:44:10.011331099Z"}
Sep 05 21:44:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:44:10.013689029Z"}
Sep 05 21:44:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:44:58.880127720Z"}
Sep 05 21:45:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:45:10.098063737Z"}
Sep 05 21:45:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:45:10.155721661Z"}
Sep 05 21:45:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:45:10.155828278Z"}
Sep 05 21:45:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:45:58.880752526Z"}
Sep 05 21:46:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:46:10.243968458Z"}
Sep 05 21:46:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:46:10.305993217Z"}
Sep 05 21:46:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:46:10.313527865Z"}
Sep 05 21:46:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","PID":23463,"State":"Cancelled","level":"info","msg":"container finished -- dropping from queue","time":"2022-09-05T21:46:10.537127534Z"}
Sep 05 21:46:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-hj58uzx8txowcgy","PID":23463,"State":"Cancelled","level":"info","msg":"dropping container from queue","time":"2022-09-05T21:46:10.537182983Z"}
Sep 05 21:46:14 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"Priority":562948291008091902,"State":"Queued","level":"info","msg":"adding container to queue","time":"2022-09-05T21:46:14.801358192Z"}
Sep 05 21:46:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:46:58.881494427Z"}
Sep 05 21:47:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:47:10.391469080Z"}
Sep 05 21:47:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:47:10.452296709Z"}
Sep 05 21:47:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:47:10.452463191Z"}
Sep 05 21:47:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:47:58.882116779Z"}
Sep 05 21:48:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:48:10.523595630Z"}
Sep 05 21:48:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:48:10.564103461Z"}
Sep 05 21:48:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:48:10.612886689Z"}
Sep 05 21:48:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:48:58.883219014Z"}
Sep 05 21:49:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:49:10.683777674Z"}
Sep 05 21:49:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:49:10.719801134Z"}
Sep 05 21:49:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:49:10.719924690Z"}
Sep 05 21:49:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:49:58.884665391Z"}
Sep 05 21:50:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:50:10.794297782Z"}
Sep 05 21:50:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:50:10.830300543Z"}
Sep 05 21:50:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:50:10.830430799Z"}
Sep 05 21:50:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:50:58.885736286Z"}
Sep 05 21:51:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:51:10.906573704Z"}
Sep 05 21:51:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","InstanceType":"localhost","PID":23463,"level":"info","msg":"creating new instance","time":"2022-09-05T21:51:10.934605924Z"}
Sep 05 21:51:10 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","InstanceType":"localhost","PID":23463,"error":"loopback driver is always at quota","level":"error","msg":"create failed","time":"2022-09-05T21:51:10.934717651Z"}
Sep 05 21:51:11 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","PID":23463,"State":"Cancelled","level":"info","msg":"container finished -- dropping from queue","time":"2022-09-05T21:51:11.080144717Z"}
Sep 05 21:51:11 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","ContainerUUID":"arva3-dz642-rml5kxd8yl0qsej","PID":23463,"State":"Cancelled","level":"info","msg":"dropping container from queue","time":"2022-09-05T21:51:11.080231263Z"}
Sep 05 21:51:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"level":"info","msg":"worker still listed after shutdown; retrying","time":"2022-09-05T21:51:58.886024352Z"}
Sep 05 21:52:58 arv-demo arvados-server[23258]: [dispatch-cloud] {"ClusterID":"arva3","Instance":"localhost","PID":23463,"WorkerState":"shutdown","level":"info","msg":"instance disappeared in cloud","time":"2022-09-05T21:52:58.887131300Z"}